2024-12-03 04:59:31,239 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 04:59:31,254 main DEBUG Took 0.013466 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 04:59:31,255 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 04:59:31,255 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 04:59:31,256 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 04:59:31,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,268 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 04:59:31,290 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,292 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,292 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,293 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,294 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,294 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,295 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,295 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,296 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,298 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,298 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,299 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,300 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,300 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,301 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,301 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,302 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,303 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,303 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,304 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,304 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:59:31,305 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,305 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 04:59:31,307 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:59:31,309 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 04:59:31,312 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 04:59:31,313 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 04:59:31,315 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 04:59:31,315 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 04:59:31,327 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 04:59:31,330 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 04:59:31,332 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 04:59:31,332 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 04:59:31,333 main DEBUG createAppenders(={Console}) 2024-12-03 04:59:31,334 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-03 04:59:31,334 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 04:59:31,335 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-03 04:59:31,335 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 04:59:31,336 main DEBUG OutputStream closed 2024-12-03 04:59:31,336 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 04:59:31,336 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 04:59:31,337 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-03 04:59:31,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 04:59:31,449 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 04:59:31,451 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 04:59:31,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 04:59:31,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 04:59:31,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 04:59:31,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 04:59:31,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 04:59:31,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 04:59:31,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 04:59:31,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 04:59:31,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 04:59:31,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 04:59:31,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 04:59:31,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 04:59:31,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 04:59:31,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 04:59:31,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 04:59:31,461 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 04:59:31,462 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-03 04:59:31,462 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 04:59:31,463 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-03T04:59:31,703 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b 2024-12-03 04:59:31,706 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 04:59:31,707 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T04:59:31,719 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-12-03T04:59:31,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T04:59:31,766 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e, deleteOnExit=true 2024-12-03T04:59:31,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T04:59:31,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/test.cache.data in system properties and HBase conf 2024-12-03T04:59:31,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T04:59:31,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/hadoop.log.dir in system properties and HBase conf 2024-12-03T04:59:31,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T04:59:31,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T04:59:31,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T04:59:31,915 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T04:59:32,049 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T04:59:32,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T04:59:32,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T04:59:32,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T04:59:32,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T04:59:32,057 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T04:59:32,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T04:59:32,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T04:59:32,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T04:59:32,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T04:59:32,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/nfs.dump.dir in system properties and HBase conf 2024-12-03T04:59:32,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/java.io.tmpdir in system properties and HBase conf 2024-12-03T04:59:32,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T04:59:32,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T04:59:32,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T04:59:33,042 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T04:59:33,162 INFO [Time-limited test {}] log.Log(170): Logging initialized @2870ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T04:59:33,238 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T04:59:33,301 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T04:59:33,321 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T04:59:33,322 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T04:59:33,323 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T04:59:33,336 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T04:59:33,339 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/hadoop.log.dir/,AVAILABLE} 2024-12-03T04:59:33,339 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T04:59:33,585 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/java.io.tmpdir/jetty-localhost-42397-hadoop-hdfs-3_4_1-tests_jar-_-any-4393528650129229784/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T04:59:33,593 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:42397} 2024-12-03T04:59:33,593 INFO [Time-limited test {}] server.Server(415): Started @3302ms 2024-12-03T04:59:34,091 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T04:59:34,098 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T04:59:34,099 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T04:59:34,100 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T04:59:34,100 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T04:59:34,101 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/hadoop.log.dir/,AVAILABLE} 2024-12-03T04:59:34,102 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T04:59:34,256 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/java.io.tmpdir/jetty-localhost-39265-hadoop-hdfs-3_4_1-tests_jar-_-any-2780189530884812018/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T04:59:34,257 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39265} 2024-12-03T04:59:34,257 INFO [Time-limited test {}] server.Server(415): Started @3966ms 2024-12-03T04:59:34,320 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T04:59:34,823 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e/data/data2/current/BP-1710912754-172.17.0.3-1733201972772/current, will proceed with Du for space computation calculation, 2024-12-03T04:59:34,823 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e/data/data1/current/BP-1710912754-172.17.0.3-1733201972772/current, will proceed with Du for space computation calculation, 2024-12-03T04:59:34,861 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T04:59:34,914 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x137563b357893d95 with lease ID 0x8d7a0589bb263bad: Processing first storage report for DS-ce700575-fd88-4a57-834e-bfe479e14bde from datanode DatanodeRegistration(127.0.0.1:41841, datanodeUuid=414d735e-106e-42de-90fe-a294762bc756, infoPort=35341, infoSecurePort=0, ipcPort=37699, storageInfo=lv=-57;cid=testClusterID;nsid=360348386;c=1733201972772) 2024-12-03T04:59:34,916 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x137563b357893d95 with lease ID 0x8d7a0589bb263bad: from storage DS-ce700575-fd88-4a57-834e-bfe479e14bde node DatanodeRegistration(127.0.0.1:41841, datanodeUuid=414d735e-106e-42de-90fe-a294762bc756, infoPort=35341, infoSecurePort=0, ipcPort=37699, storageInfo=lv=-57;cid=testClusterID;nsid=360348386;c=1733201972772), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T04:59:34,916 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x137563b357893d95 with lease ID 0x8d7a0589bb263bad: Processing first storage report for DS-d615e491-dc5e-45bf-ad44-ec55de464035 from datanode DatanodeRegistration(127.0.0.1:41841, datanodeUuid=414d735e-106e-42de-90fe-a294762bc756, infoPort=35341, infoSecurePort=0, ipcPort=37699, storageInfo=lv=-57;cid=testClusterID;nsid=360348386;c=1733201972772) 2024-12-03T04:59:34,917 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x137563b357893d95 with lease ID 0x8d7a0589bb263bad: from storage DS-d615e491-dc5e-45bf-ad44-ec55de464035 node DatanodeRegistration(127.0.0.1:41841, datanodeUuid=414d735e-106e-42de-90fe-a294762bc756, infoPort=35341, infoSecurePort=0, ipcPort=37699, storageInfo=lv=-57;cid=testClusterID;nsid=360348386;c=1733201972772), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T04:59:34,934 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b 2024-12-03T04:59:35,023 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e/zookeeper_0, clientPort=52673, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T04:59:35,035 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52673 2024-12-03T04:59:35,047 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:59:35,050 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:59:35,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741825_1001 (size=7) 2024-12-03T04:59:35,798 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d with version=8 2024-12-03T04:59:35,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/hbase-staging 2024-12-03T04:59:35,951 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T04:59:36,335 INFO [Time-limited test {}] client.ConnectionUtils(128): master/32beccd76b35:0 server-side Connection retries=45 2024-12-03T04:59:36,350 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T04:59:36,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T04:59:36,360 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T04:59:36,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T04:59:36,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T04:59:36,583 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T04:59:36,694 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T04:59:36,707 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T04:59:36,713 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T04:59:36,752 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5558 (auto-detected) 2024-12-03T04:59:36,754 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T04:59:36,781 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43423 2024-12-03T04:59:36,815 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43423 connecting to ZooKeeper ensemble=127.0.0.1:52673 2024-12-03T04:59:36,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:434230x0, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T04:59:36,863 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43423-0x10151764b860000 connected 2024-12-03T04:59:36,927 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:59:36,930 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:59:36,942 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T04:59:36,947 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d, hbase.cluster.distributed=false 2024-12-03T04:59:36,982 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T04:59:36,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43423 2024-12-03T04:59:36,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43423 2024-12-03T04:59:37,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43423 2024-12-03T04:59:37,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43423 2024-12-03T04:59:37,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43423 2024-12-03T04:59:37,159 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32beccd76b35:0 server-side Connection retries=45 2024-12-03T04:59:37,162 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T04:59:37,163 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T04:59:37,163 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T04:59:37,164 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T04:59:37,164 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T04:59:37,167 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T04:59:37,171 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T04:59:37,173 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34719 2024-12-03T04:59:37,175 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34719 connecting to ZooKeeper ensemble=127.0.0.1:52673 2024-12-03T04:59:37,177 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:59:37,182 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:59:37,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347190x0, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T04:59:37,206 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34719-0x10151764b860001 connected 2024-12-03T04:59:37,208 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T04:59:37,214 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T04:59:37,233 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T04:59:37,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T04:59:37,248 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T04:59:37,253 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34719 2024-12-03T04:59:37,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34719 2024-12-03T04:59:37,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34719 2024-12-03T04:59:37,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34719 2024-12-03T04:59:37,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34719 2024-12-03T04:59:37,282 DEBUG [M:0;32beccd76b35:43423 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;32beccd76b35:43423 2024-12-03T04:59:37,283 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/32beccd76b35,43423,1733201976038 2024-12-03T04:59:37,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T04:59:37,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T04:59:37,293 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/32beccd76b35,43423,1733201976038 2024-12-03T04:59:37,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:37,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T04:59:37,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:37,320 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T04:59:37,322 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/32beccd76b35,43423,1733201976038 from backup master directory 2024-12-03T04:59:37,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/32beccd76b35,43423,1733201976038 2024-12-03T04:59:37,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T04:59:37,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T04:59:37,328 WARN [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T04:59:37,328 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=32beccd76b35,43423,1733201976038 2024-12-03T04:59:37,331 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T04:59:37,332 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T04:59:37,419 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/hbase.id] with ID: 2f04a841-bf4e-4eda-85ec-19b4a03d3411 2024-12-03T04:59:37,419 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/.tmp/hbase.id 2024-12-03T04:59:37,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741826_1002 (size=42) 2024-12-03T04:59:37,441 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/.tmp/hbase.id]:[hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/hbase.id] 2024-12-03T04:59:37,501 INFO [master/32beccd76b35:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:59:37,507 INFO [master/32beccd76b35:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T04:59:37,532 INFO [master/32beccd76b35:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-12-03T04:59:37,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:37,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:37,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741827_1003 (size=196) 2024-12-03T04:59:37,578 INFO [master/32beccd76b35:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:59:37,582 INFO [master/32beccd76b35:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T04:59:37,605 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:37,611 INFO [master/32beccd76b35:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T04:59:37,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741828_1004 (size=1189) 2024-12-03T04:59:37,689 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/MasterData/data/master/store 2024-12-03T04:59:37,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741829_1005 (size=34) 2024-12-03T04:59:38,158 INFO [master/32beccd76b35:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T04:59:38,162 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:59:38,163 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T04:59:38,164 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:59:38,164 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:59:38,166 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T04:59:38,166 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:59:38,167 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:59:38,168 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733201978163Disabling compacts and flushes for region at 1733201978163Disabling writes for close at 1733201978166 (+3 ms)Writing region close event to WAL at 1733201978167 (+1 ms)Closed at 1733201978167 2024-12-03T04:59:38,171 WARN [master/32beccd76b35:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/MasterData/data/master/store/.initializing 2024-12-03T04:59:38,171 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/MasterData/WALs/32beccd76b35,43423,1733201976038 2024-12-03T04:59:38,182 INFO [master/32beccd76b35:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T04:59:38,200 INFO [master/32beccd76b35:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32beccd76b35%2C43423%2C1733201976038, suffix=, logDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/MasterData/WALs/32beccd76b35,43423,1733201976038, archiveDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/MasterData/oldWALs, maxLogs=10 2024-12-03T04:59:38,231 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/MasterData/WALs/32beccd76b35,43423,1733201976038/32beccd76b35%2C43423%2C1733201976038.1733201978206, exclude list is [], retry=0 2024-12-03T04:59:38,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41841,DS-ce700575-fd88-4a57-834e-bfe479e14bde,DISK] 2024-12-03T04:59:38,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T04:59:38,303 INFO [master/32beccd76b35:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/MasterData/WALs/32beccd76b35,43423,1733201976038/32beccd76b35%2C43423%2C1733201976038.1733201978206 2024-12-03T04:59:38,304 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35341:35341)] 2024-12-03T04:59:38,305 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:59:38,305 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:59:38,310 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,311 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,356 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,392 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T04:59:38,397 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:38,400 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:59:38,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,405 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T04:59:38,405 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:38,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:59:38,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,411 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T04:59:38,411 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:38,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:59:38,413 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,416 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T04:59:38,416 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:38,418 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:59:38,418 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,422 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,423 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,430 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,431 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,436 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T04:59:38,441 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:59:38,450 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T04:59:38,451 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62180618, jitterRate=-0.07343658804893494}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T04:59:38,458 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733201978327Initializing all the Stores at 1733201978330 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733201978330Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201978331 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201978332 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201978332Cleaning up temporary data from old regions at 1733201978431 (+99 ms)Region opened successfully at 1733201978457 (+26 ms) 2024-12-03T04:59:38,459 INFO [master/32beccd76b35:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T04:59:38,501 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d907c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32beccd76b35/172.17.0.3:0 2024-12-03T04:59:38,541 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T04:59:38,556 INFO [master/32beccd76b35:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T04:59:38,556 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T04:59:38,559 INFO [master/32beccd76b35:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T04:59:38,561 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T04:59:38,566 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-03T04:59:38,566 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T04:59:38,597 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T04:59:38,607 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T04:59:38,610 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T04:59:38,612 INFO [master/32beccd76b35:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T04:59:38,614 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T04:59:38,616 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T04:59:38,618 INFO [master/32beccd76b35:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T04:59:38,622 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T04:59:38,624 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T04:59:38,626 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T04:59:38,628 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T04:59:38,649 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T04:59:38,651 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T04:59:38,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T04:59:38,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T04:59:38,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:38,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:38,659 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=32beccd76b35,43423,1733201976038, sessionid=0x10151764b860000, setting cluster-up flag (Was=false) 2024-12-03T04:59:38,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:38,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:38,679 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T04:59:38,681 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32beccd76b35,43423,1733201976038 2024-12-03T04:59:38,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:38,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:38,695 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T04:59:38,697 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32beccd76b35,43423,1733201976038 2024-12-03T04:59:38,705 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T04:59:38,769 INFO [RS:0;32beccd76b35:34719 {}] regionserver.HRegionServer(746): ClusterId : 2f04a841-bf4e-4eda-85ec-19b4a03d3411 2024-12-03T04:59:38,773 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T04:59:38,779 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T04:59:38,779 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T04:59:38,786 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T04:59:38,787 DEBUG [RS:0;32beccd76b35:34719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62411ec8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32beccd76b35/172.17.0.3:0 2024-12-03T04:59:38,803 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T04:59:38,806 DEBUG [RS:0;32beccd76b35:34719 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;32beccd76b35:34719 2024-12-03T04:59:38,811 INFO [RS:0;32beccd76b35:34719 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T04:59:38,811 INFO [RS:0;32beccd76b35:34719 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T04:59:38,811 DEBUG [RS:0;32beccd76b35:34719 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T04:59:38,814 INFO [RS:0;32beccd76b35:34719 {}] regionserver.HRegionServer(2659): reportForDuty to master=32beccd76b35,43423,1733201976038 with port=34719, startcode=1733201977102 2024-12-03T04:59:38,814 INFO [master/32beccd76b35:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T04:59:38,822 INFO [master/32beccd76b35:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T04:59:38,828 DEBUG [RS:0;32beccd76b35:34719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T04:59:38,828 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 32beccd76b35,43423,1733201976038 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T04:59:38,837 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/32beccd76b35:0, corePoolSize=5, maxPoolSize=5 2024-12-03T04:59:38,837 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/32beccd76b35:0, corePoolSize=5, maxPoolSize=5 2024-12-03T04:59:38,837 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/32beccd76b35:0, corePoolSize=5, maxPoolSize=5 2024-12-03T04:59:38,838 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/32beccd76b35:0, corePoolSize=5, maxPoolSize=5 2024-12-03T04:59:38,838 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/32beccd76b35:0, corePoolSize=10, maxPoolSize=10 2024-12-03T04:59:38,838 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:38,839 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/32beccd76b35:0, corePoolSize=2, maxPoolSize=2 2024-12-03T04:59:38,839 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:38,846 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T04:59:38,847 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T04:59:38,855 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:38,862 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733202008862 2024-12-03T04:59:38,863 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T04:59:38,864 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T04:59:38,870 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T04:59:38,856 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T04:59:38,870 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T04:59:38,871 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T04:59:38,871 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T04:59:38,874 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:38,877 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T04:59:38,878 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T04:59:38,879 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T04:59:38,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741831_1007 (size=1321) 2024-12-03T04:59:38,885 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T04:59:38,888 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T04:59:38,888 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d 2024-12-03T04:59:38,889 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T04:59:38,890 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42343, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T04:59:38,891 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/32beccd76b35:0:becomeActiveMaster-HFileCleaner.large.0-1733201978891,5,FailOnTimeoutGroup] 2024-12-03T04:59:38,892 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/32beccd76b35:0:becomeActiveMaster-HFileCleaner.small.0-1733201978891,5,FailOnTimeoutGroup] 2024-12-03T04:59:38,892 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:38,892 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T04:59:38,893 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:38,894 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:38,899 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43423 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32beccd76b35,34719,1733201977102 2024-12-03T04:59:38,901 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43423 {}] master.ServerManager(517): Registering regionserver=32beccd76b35,34719,1733201977102 2024-12-03T04:59:38,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741832_1008 (size=32) 2024-12-03T04:59:38,907 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:59:38,918 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T04:59:38,921 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T04:59:38,921 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:38,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:59:38,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T04:59:38,922 DEBUG [RS:0;32beccd76b35:34719 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d 2024-12-03T04:59:38,923 DEBUG [RS:0;32beccd76b35:34719 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37421 2024-12-03T04:59:38,923 DEBUG [RS:0;32beccd76b35:34719 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T04:59:38,925 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T04:59:38,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:38,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:59:38,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T04:59:38,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T04:59:38,929 DEBUG [RS:0;32beccd76b35:34719 {}] zookeeper.ZKUtil(111): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32beccd76b35,34719,1733201977102 2024-12-03T04:59:38,929 WARN [RS:0;32beccd76b35:34719 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T04:59:38,930 INFO [RS:0;32beccd76b35:34719 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T04:59:38,930 DEBUG [RS:0;32beccd76b35:34719 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/WALs/32beccd76b35,34719,1733201977102 2024-12-03T04:59:38,930 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T04:59:38,930 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:38,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:59:38,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T04:59:38,935 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T04:59:38,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:38,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:59:38,936 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T04:59:38,938 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740 2024-12-03T04:59:38,938 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740 2024-12-03T04:59:38,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T04:59:38,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T04:59:38,943 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32beccd76b35,34719,1733201977102] 2024-12-03T04:59:38,943 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:59:38,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T04:59:38,952 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T04:59:38,954 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65415315, jitterRate=-0.025235846638679504}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:59:38,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733201978908Initializing all the Stores at 1733201978910 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733201978910Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733201978917 (+7 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201978917Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733201978918 (+1 ms)Cleaning up temporary data from old regions at 1733201978942 (+24 ms)Region opened successfully at 1733201978958 (+16 ms) 2024-12-03T04:59:38,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T04:59:38,959 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T04:59:38,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T04:59:38,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T04:59:38,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T04:59:38,961 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T04:59:38,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733201978959Disabling compacts and flushes for region at 1733201978959Disabling writes for close at 1733201978959Writing region close event to WAL at 1733201978961 (+2 ms)Closed at 1733201978961 2024-12-03T04:59:38,966 INFO [RS:0;32beccd76b35:34719 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T04:59:38,966 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T04:59:38,966 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T04:59:38,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T04:59:38,986 INFO [RS:0;32beccd76b35:34719 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T04:59:38,993 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T04:59:38,995 INFO [RS:0;32beccd76b35:34719 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T04:59:38,995 INFO [RS:0;32beccd76b35:34719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:38,997 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T04:59:38,998 INFO [RS:0;32beccd76b35:34719 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T04:59:39,009 INFO [RS:0;32beccd76b35:34719 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T04:59:39,011 INFO [RS:0;32beccd76b35:34719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,013 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:39,014 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:39,014 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:39,014 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:39,014 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:39,015 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32beccd76b35:0, corePoolSize=2, maxPoolSize=2 2024-12-03T04:59:39,015 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:39,015 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:39,015 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:39,015 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:39,016 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:39,016 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:59:39,016 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32beccd76b35:0, corePoolSize=3, maxPoolSize=3 2024-12-03T04:59:39,016 DEBUG [RS:0;32beccd76b35:34719 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0, corePoolSize=3, maxPoolSize=3 2024-12-03T04:59:39,018 INFO [RS:0;32beccd76b35:34719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,018 INFO [RS:0;32beccd76b35:34719 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,018 INFO [RS:0;32beccd76b35:34719 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,018 INFO [RS:0;32beccd76b35:34719 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,019 INFO [RS:0;32beccd76b35:34719 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,019 INFO [RS:0;32beccd76b35:34719 {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,34719,1733201977102-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T04:59:39,043 INFO [RS:0;32beccd76b35:34719 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T04:59:39,046 INFO [RS:0;32beccd76b35:34719 {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,34719,1733201977102-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,047 INFO [RS:0;32beccd76b35:34719 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,047 INFO [RS:0;32beccd76b35:34719 {}] regionserver.Replication(171): 32beccd76b35,34719,1733201977102 started 2024-12-03T04:59:39,071 INFO [RS:0;32beccd76b35:34719 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,072 INFO [RS:0;32beccd76b35:34719 {}] regionserver.HRegionServer(1482): Serving as 32beccd76b35,34719,1733201977102, RpcServer on 32beccd76b35/172.17.0.3:34719, sessionid=0x10151764b860001 2024-12-03T04:59:39,073 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T04:59:39,073 DEBUG [RS:0;32beccd76b35:34719 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32beccd76b35,34719,1733201977102 2024-12-03T04:59:39,074 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32beccd76b35,34719,1733201977102' 2024-12-03T04:59:39,074 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T04:59:39,077 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T04:59:39,078 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T04:59:39,078 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T04:59:39,078 DEBUG [RS:0;32beccd76b35:34719 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32beccd76b35,34719,1733201977102 2024-12-03T04:59:39,078 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32beccd76b35,34719,1733201977102' 2024-12-03T04:59:39,078 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T04:59:39,079 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T04:59:39,081 DEBUG [RS:0;32beccd76b35:34719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T04:59:39,081 INFO [RS:0;32beccd76b35:34719 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T04:59:39,081 INFO [RS:0;32beccd76b35:34719 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T04:59:39,148 WARN [32beccd76b35:43423 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T04:59:39,188 INFO [RS:0;32beccd76b35:34719 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T04:59:39,192 INFO [RS:0;32beccd76b35:34719 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32beccd76b35%2C34719%2C1733201977102, suffix=, logDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/WALs/32beccd76b35,34719,1733201977102, archiveDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/oldWALs, maxLogs=32 2024-12-03T04:59:39,212 DEBUG [RS:0;32beccd76b35:34719 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/WALs/32beccd76b35,34719,1733201977102/32beccd76b35%2C34719%2C1733201977102.1733201979195, exclude list is [], retry=0 2024-12-03T04:59:39,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41841,DS-ce700575-fd88-4a57-834e-bfe479e14bde,DISK] 2024-12-03T04:59:39,223 INFO [RS:0;32beccd76b35:34719 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/WALs/32beccd76b35,34719,1733201977102/32beccd76b35%2C34719%2C1733201977102.1733201979195 2024-12-03T04:59:39,223 DEBUG [RS:0;32beccd76b35:34719 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35341:35341)] 2024-12-03T04:59:39,402 DEBUG [32beccd76b35:43423 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T04:59:39,416 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T04:59:39,425 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32beccd76b35,34719,1733201977102, state=OPENING 2024-12-03T04:59:39,432 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T04:59:39,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:39,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:59:39,436 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T04:59:39,436 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T04:59:39,437 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T04:59:39,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32beccd76b35,34719,1733201977102}] 2024-12-03T04:59:39,623 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T04:59:39,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54547, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T04:59:39,644 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T04:59:39,644 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T04:59:39,645 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T04:59:39,650 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32beccd76b35%2C34719%2C1733201977102.meta, suffix=.meta, logDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/WALs/32beccd76b35,34719,1733201977102, archiveDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/oldWALs, maxLogs=32 2024-12-03T04:59:39,675 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/WALs/32beccd76b35,34719,1733201977102/32beccd76b35%2C34719%2C1733201977102.meta.1733201979652.meta, exclude list is [], retry=0 2024-12-03T04:59:39,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41841,DS-ce700575-fd88-4a57-834e-bfe479e14bde,DISK] 2024-12-03T04:59:39,684 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/WALs/32beccd76b35,34719,1733201977102/32beccd76b35%2C34719%2C1733201977102.meta.1733201979652.meta 2024-12-03T04:59:39,685 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35341:35341)] 2024-12-03T04:59:39,685 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:59:39,688 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T04:59:39,691 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T04:59:39,697 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T04:59:39,703 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T04:59:39,703 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:59:39,704 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T04:59:39,704 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T04:59:39,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T04:59:39,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T04:59:39,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:39,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:59:39,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T04:59:39,713 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T04:59:39,713 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:39,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:59:39,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T04:59:39,716 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T04:59:39,716 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:39,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:59:39,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T04:59:39,719 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T04:59:39,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:39,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:59:39,720 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T04:59:39,721 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740 2024-12-03T04:59:39,724 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740 2024-12-03T04:59:39,727 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T04:59:39,727 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T04:59:39,729 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:59:39,732 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T04:59:39,734 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69245997, jitterRate=0.03184576332569122}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:59:39,734 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T04:59:39,736 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733201979704Writing region info on filesystem at 1733201979705 (+1 ms)Initializing all the Stores at 1733201979707 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733201979707Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733201979707Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201979707Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733201979707Cleaning up temporary data from old regions at 1733201979728 (+21 ms)Running coprocessor post-open hooks at 1733201979734 (+6 ms)Region opened successfully at 1733201979736 (+2 ms) 2024-12-03T04:59:39,744 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733201979612 2024-12-03T04:59:39,757 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T04:59:39,757 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T04:59:39,760 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T04:59:39,762 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32beccd76b35,34719,1733201977102, state=OPEN 2024-12-03T04:59:39,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34719-0x10151764b860001, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T04:59:39,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10151764b860000, quorum=127.0.0.1:52673, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T04:59:39,768 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T04:59:39,768 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T04:59:39,768 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:39,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T04:59:39,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32beccd76b35,34719,1733201977102 in 329 msec 2024-12-03T04:59:39,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T04:59:39,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 802 msec 2024-12-03T04:59:39,784 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T04:59:39,784 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T04:59:39,811 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:39,813 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:39,842 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:39,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34185, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:39,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1280 sec 2024-12-03T04:59:39,881 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733201979880, completionTime=-1 2024-12-03T04:59:39,885 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T04:59:39,885 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T04:59:39,925 INFO [master/32beccd76b35:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T04:59:39,925 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733202039925 2024-12-03T04:59:39,926 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733202099926 2024-12-03T04:59:39,926 INFO [master/32beccd76b35:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 40 msec 2024-12-03T04:59:39,929 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,43423,1733201976038-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,929 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,43423,1733201976038-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,930 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,43423,1733201976038-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,932 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-32beccd76b35:43423, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,932 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,933 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:39,941 DEBUG [master/32beccd76b35:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T04:59:39,972 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.643sec 2024-12-03T04:59:39,974 INFO [master/32beccd76b35:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T04:59:39,975 INFO [master/32beccd76b35:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T04:59:39,977 INFO [master/32beccd76b35:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T04:59:39,978 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T04:59:39,978 INFO [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T04:59:39,979 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,43423,1733201976038-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T04:59:39,980 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,43423,1733201976038-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T04:59:40,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:40,036 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T04:59:40,037 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T04:59:40,040 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T04:59:40,041 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T04:59:40,041 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,43423,1733201976038-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:59:40,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T04:59:40,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:59:40,091 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T04:59:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:59:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T04:59:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f93b2f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T04:59:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:59:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:59:40,106 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:59:40,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:40,112 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:40,132 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:40,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=32beccd76b35,43423,1733201976038 2024-12-03T04:59:40,179 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=629, ProcessCount=11, AvailableMemoryMB=4524 2024-12-03T04:59:40,192 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T04:59:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,43423,1733201976038 2024-12-03T04:59:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a16ee17 2024-12-03T04:59:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T04:59:40,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T04:59:40,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T04:59:40,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:59:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T04:59:40,234 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T04:59:40,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-03T04:59:40,236 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:40,239 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T04:59:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T04:59:40,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741835_1011 (size=959) 2024-12-03T04:59:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T04:59:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T04:59:40,676 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d 2024-12-03T04:59:40,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741836_1012 (size=53) 2024-12-03T04:59:40,688 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:59:40,689 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1fc1c9cce64fed6eabb56af3d145cb45, disabling compactions & flushes 2024-12-03T04:59:40,689 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:40,689 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:40,689 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. after waiting 0 ms 2024-12-03T04:59:40,689 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:40,689 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:40,689 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1fc1c9cce64fed6eabb56af3d145cb45: Waiting for close lock at 1733201980689Disabling compacts and flushes for region at 1733201980689Disabling writes for close at 1733201980689Writing region close event to WAL at 1733201980689Closed at 1733201980689 2024-12-03T04:59:40,692 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T04:59:40,699 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733201980692"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733201980692"}]},"ts":"1733201980692"} 2024-12-03T04:59:40,706 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T04:59:40,708 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T04:59:40,712 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201980709"}]},"ts":"1733201980709"} 2024-12-03T04:59:40,717 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T04:59:40,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fc1c9cce64fed6eabb56af3d145cb45, ASSIGN}] 2024-12-03T04:59:40,722 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fc1c9cce64fed6eabb56af3d145cb45, ASSIGN 2024-12-03T04:59:40,725 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fc1c9cce64fed6eabb56af3d145cb45, ASSIGN; state=OFFLINE, location=32beccd76b35,34719,1733201977102; forceNewPlan=false, retain=false 2024-12-03T04:59:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T04:59:40,877 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1fc1c9cce64fed6eabb56af3d145cb45, regionState=OPENING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T04:59:40,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fc1c9cce64fed6eabb56af3d145cb45, ASSIGN because future has completed 2024-12-03T04:59:40,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102}] 2024-12-03T04:59:41,048 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:41,049 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:59:41,049 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,050 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:59:41,050 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,050 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,053 INFO [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,056 INFO [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fc1c9cce64fed6eabb56af3d145cb45 columnFamilyName A 2024-12-03T04:59:41,056 DEBUG [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:41,057 INFO [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] regionserver.HStore(327): Store=1fc1c9cce64fed6eabb56af3d145cb45/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:59:41,058 INFO [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,062 INFO [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fc1c9cce64fed6eabb56af3d145cb45 columnFamilyName B 2024-12-03T04:59:41,062 DEBUG [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:41,063 INFO [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] regionserver.HStore(327): Store=1fc1c9cce64fed6eabb56af3d145cb45/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:59:41,063 INFO [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,065 INFO [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fc1c9cce64fed6eabb56af3d145cb45 columnFamilyName C 2024-12-03T04:59:41,066 DEBUG [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:59:41,067 INFO [StoreOpener-1fc1c9cce64fed6eabb56af3d145cb45-1 {}] regionserver.HStore(327): Store=1fc1c9cce64fed6eabb56af3d145cb45/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:59:41,067 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,068 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,069 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,072 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,072 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,073 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:59:41,077 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,083 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T04:59:41,084 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1fc1c9cce64fed6eabb56af3d145cb45; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66271272, jitterRate=-0.012481093406677246}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:59:41,084 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,086 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1fc1c9cce64fed6eabb56af3d145cb45: Running coprocessor pre-open hook at 1733201981050Writing region info on filesystem at 1733201981050Initializing all the Stores at 1733201981052 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201981052Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201981052Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201981052Cleaning up temporary data from old regions at 1733201981072 (+20 ms)Running coprocessor post-open hooks at 1733201981084 (+12 ms)Region opened successfully at 1733201981086 (+2 ms) 2024-12-03T04:59:41,088 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., pid=6, masterSystemTime=1733201981040 2024-12-03T04:59:41,093 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:41,093 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:41,094 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1fc1c9cce64fed6eabb56af3d145cb45, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T04:59:41,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T04:59:41,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T04:59:41,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 in 220 msec 2024-12-03T04:59:41,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T04:59:41,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fc1c9cce64fed6eabb56af3d145cb45, ASSIGN in 398 msec 2024-12-03T04:59:41,131 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T04:59:41,132 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201981131"}]},"ts":"1733201981131"} 2024-12-03T04:59:41,147 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T04:59:41,152 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T04:59:41,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 931 msec 2024-12-03T04:59:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T04:59:41,383 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:59:41,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T04:59:41,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:59:41,402 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T04:59:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:59:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T04:59:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d463185, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T04:59:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:59:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:59:41,415 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:59:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f0fa42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,418 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:41,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:41,433 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T04:59:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,43423,1733201976038 2024-12-03T04:59:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64ba8396 2024-12-03T04:59:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T04:59:41,441 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T04:59:41,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T04:59:41,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:59:41,451 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T04:59:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:59:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T04:59:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@470128a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T04:59:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:59:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:59:41,455 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:59:41,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@174557e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,461 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T04:59:41,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:59:41,465 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T04:59:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:59:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T04:59:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d21add1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T04:59:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:59:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:59:41,472 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:59:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d4406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,478 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T04:59:41,478 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:59:41,479 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T04:59:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:59:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T04:59:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740f65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T04:59:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:59:41,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:59:41,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:59:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T04:59:41,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:59:41,487 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T04:59:41,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:59:41,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T04:59:41,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T04:59:41,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:59:41,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:59:41,490 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:59:41,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253fe80a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T04:59:41,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:59:41,495 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T04:59:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:59:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T04:59:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e3ab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T04:59:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:59:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:59:41,498 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:59:41,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c4ddcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T04:59:41,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:59:41,502 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T04:59:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:59:41,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T04:59:41,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38cde2e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T04:59:41,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:59:41,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:59:41,505 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:59:41,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d4dbff7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T04:59:41,508 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:59:41,509 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T04:59:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:59:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T04:59:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6396285d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T04:59:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:59:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:59:41,511 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:59:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@439f6406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T04:59:41,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:59:41,519 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T04:59:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:59:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T04:59:41,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92f3d3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T04:59:41,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:59:41,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:59:41,522 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:59:41,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ee4412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,530 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T04:59:41,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:59:41,540 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T04:59:41,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:59:41,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T04:59:41,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cba810e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T04:59:41,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:59:41,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:59:41,544 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:59:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c8f202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:59:41,555 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:41,556 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:41,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:41,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:41,568 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:41,569 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:41,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:41,570 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T04:59:41,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:41,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:41,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:41,574 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:41,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:41,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:41,586 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T04:59:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T04:59:41,590 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:41,591 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:59:41,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:41,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:41,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:41,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T04:59:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T04:59:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:59:41,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:41,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:59:41,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:59:41,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T04:59:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-03T04:59:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T04:59:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T04:59:41,619 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:59:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:59:41,621 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:59:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T04:59:41,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:59:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T04:59:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:59:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:41,781 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:41,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T04:59:41,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:59:41,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:41,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:41,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:59:41,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/3d7f0add986f4f69a33b4b6554c73172 is 50, key is test_row_1/A:col10/1733201981754/Put/seqid=0 2024-12-03T04:59:42,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:59:42,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:42,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202042008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741837_1013 (size=11997) 2024-12-03T04:59:42,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/3d7f0add986f4f69a33b4b6554c73172 2024-12-03T04:59:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202042026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202042039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202042042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202042045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:59:42,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:42,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/f0def0871b7443279c90cf0f9301acd1 is 50, key is test_row_1/B:col10/1733201981754/Put/seqid=0 2024-12-03T04:59:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:59:42,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202042262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202042263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202042264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202042266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202042270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741838_1014 (size=9657) 2024-12-03T04:59:42,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/f0def0871b7443279c90cf0f9301acd1 2024-12-03T04:59:42,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/5f08f6f99b7d448ab8709e186d09d5bf is 50, key is test_row_1/C:col10/1733201981754/Put/seqid=0 2024-12-03T04:59:42,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:59:42,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:42,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741839_1015 (size=9657) 2024-12-03T04:59:42,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/5f08f6f99b7d448ab8709e186d09d5bf 2024-12-03T04:59:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/3d7f0add986f4f69a33b4b6554c73172 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/3d7f0add986f4f69a33b4b6554c73172 2024-12-03T04:59:42,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/3d7f0add986f4f69a33b4b6554c73172, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T04:59:42,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/f0def0871b7443279c90cf0f9301acd1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f0def0871b7443279c90cf0f9301acd1 2024-12-03T04:59:42,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f0def0871b7443279c90cf0f9301acd1, entries=100, sequenceid=13, filesize=9.4 K 2024-12-03T04:59:42,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/5f08f6f99b7d448ab8709e186d09d5bf as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/5f08f6f99b7d448ab8709e186d09d5bf 2024-12-03T04:59:42,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/5f08f6f99b7d448ab8709e186d09d5bf, entries=100, sequenceid=13, filesize=9.4 K 2024-12-03T04:59:42,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 1fc1c9cce64fed6eabb56af3d145cb45 in 725ms, sequenceid=13, compaction requested=false 2024-12-03T04:59:42,526 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T04:59:42,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:42,544 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:42,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-12-03T04:59:42,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:59:42,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:42,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202042530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202042547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202042555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/8cfe53779d944f6abc414c01d9489682 is 50, key is test_row_0/A:col10/1733201982521/Put/seqid=0 2024-12-03T04:59:42,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202042558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202042560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741840_1016 (size=12001) 2024-12-03T04:59:42,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/8cfe53779d944f6abc414c01d9489682 2024-12-03T04:59:42,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/208193c6a60348659c942e58c96996b4 is 50, key is test_row_0/B:col10/1733201982521/Put/seqid=0 2024-12-03T04:59:42,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741841_1017 (size=12001) 2024-12-03T04:59:42,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/208193c6a60348659c942e58c96996b4 2024-12-03T04:59:42,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202042672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202042673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202042673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202042701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:59:42,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202042702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/ef6e7c3c5b5747dda3e17ef837158694 is 50, key is test_row_0/C:col10/1733201982521/Put/seqid=0 2024-12-03T04:59:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:59:42,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741842_1018 (size=12001) 2024-12-03T04:59:42,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:59:42,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:42,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:42,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:42,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202042903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202042903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202042903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202042926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202042934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:59:43,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:43,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:43,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:43,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:43,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:43,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/ef6e7c3c5b5747dda3e17ef837158694 2024-12-03T04:59:43,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/8cfe53779d944f6abc414c01d9489682 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cfe53779d944f6abc414c01d9489682 2024-12-03T04:59:43,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:59:43,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:43,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:43,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:43,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:43,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:43,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cfe53779d944f6abc414c01d9489682, entries=150, sequenceid=44, filesize=11.7 K 2024-12-03T04:59:43,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/208193c6a60348659c942e58c96996b4 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/208193c6a60348659c942e58c96996b4 2024-12-03T04:59:43,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/208193c6a60348659c942e58c96996b4, entries=150, sequenceid=44, filesize=11.7 K 2024-12-03T04:59:43,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/ef6e7c3c5b5747dda3e17ef837158694 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/ef6e7c3c5b5747dda3e17ef837158694 2024-12-03T04:59:43,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/ef6e7c3c5b5747dda3e17ef837158694, entries=150, sequenceid=44, filesize=11.7 K 2024-12-03T04:59:43,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 1fc1c9cce64fed6eabb56af3d145cb45 in 701ms, sequenceid=44, compaction requested=false 2024-12-03T04:59:43,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:43,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:59:43,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:43,354 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-03T04:59:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:43,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/6b21e5248f6e499caf4ff6613f5af5ce is 50, key is test_row_0/A:col10/1733201983274/Put/seqid=0 2024-12-03T04:59:43,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741843_1019 (size=11997) 2024-12-03T04:59:43,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202043555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202043556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202043560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202043571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202043567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202043693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202043694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202043697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202043710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202043711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:59:43,882 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/6b21e5248f6e499caf4ff6613f5af5ce 2024-12-03T04:59:43,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202043915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202043915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202043923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202043923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/a48473c0360a4e09917f3aad40e74808 is 50, key is test_row_0/B:col10/1733201983274/Put/seqid=0 2024-12-03T04:59:43,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202043935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:43,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741844_1020 (size=9657) 2024-12-03T04:59:43,999 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/a48473c0360a4e09917f3aad40e74808 2024-12-03T04:59:44,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/e751cd436fee4403b5a18168d1cb8700 is 50, key is test_row_0/C:col10/1733201983274/Put/seqid=0 2024-12-03T04:59:44,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741845_1021 (size=9657) 2024-12-03T04:59:44,113 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/e751cd436fee4403b5a18168d1cb8700 2024-12-03T04:59:44,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/6b21e5248f6e499caf4ff6613f5af5ce as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6b21e5248f6e499caf4ff6613f5af5ce 2024-12-03T04:59:44,203 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6b21e5248f6e499caf4ff6613f5af5ce, entries=150, sequenceid=56, filesize=11.7 K 2024-12-03T04:59:44,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/a48473c0360a4e09917f3aad40e74808 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a48473c0360a4e09917f3aad40e74808 2024-12-03T04:59:44,218 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a48473c0360a4e09917f3aad40e74808, entries=100, sequenceid=56, filesize=9.4 K 2024-12-03T04:59:44,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/e751cd436fee4403b5a18168d1cb8700 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/e751cd436fee4403b5a18168d1cb8700 2024-12-03T04:59:44,233 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/e751cd436fee4403b5a18168d1cb8700, entries=100, sequenceid=56, filesize=9.4 K 2024-12-03T04:59:44,238 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1fc1c9cce64fed6eabb56af3d145cb45 in 884ms, sequenceid=56, compaction requested=true 2024-12-03T04:59:44,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:44,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:44,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-03T04:59:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-03T04:59:44,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T04:59:44,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6250 sec 2024-12-03T04:59:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:44,258 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:44,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T04:59:44,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.6470 sec 2024-12-03T04:59:44,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/60df560727b24328824e916e59aba2d3 is 50, key is test_row_0/A:col10/1733201984252/Put/seqid=0 2024-12-03T04:59:44,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202044278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202044283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202044294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202044296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202044301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741846_1022 (size=16681) 2024-12-03T04:59:44,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/60df560727b24328824e916e59aba2d3 2024-12-03T04:59:44,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/0ed2391fa37749d28452ef02bef5c28c is 50, key is test_row_0/B:col10/1733201984252/Put/seqid=0 2024-12-03T04:59:44,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202044411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202044413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202044414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202044415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202044416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741847_1023 (size=12001) 2024-12-03T04:59:44,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/0ed2391fa37749d28452ef02bef5c28c 2024-12-03T04:59:44,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/0013f32453354c44949dd48ead1bc28d is 50, key is test_row_0/C:col10/1733201984252/Put/seqid=0 2024-12-03T04:59:44,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741848_1024 (size=12001) 2024-12-03T04:59:44,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202044624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202044623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202044624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202044634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202044634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/0013f32453354c44949dd48ead1bc28d 2024-12-03T04:59:44,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202044946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202044947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202044947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202044953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202044961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:44,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/60df560727b24328824e916e59aba2d3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/60df560727b24328824e916e59aba2d3 2024-12-03T04:59:45,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/60df560727b24328824e916e59aba2d3, entries=250, sequenceid=82, filesize=16.3 K 2024-12-03T04:59:45,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/0ed2391fa37749d28452ef02bef5c28c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/0ed2391fa37749d28452ef02bef5c28c 2024-12-03T04:59:45,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/0ed2391fa37749d28452ef02bef5c28c, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T04:59:45,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/0013f32453354c44949dd48ead1bc28d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0013f32453354c44949dd48ead1bc28d 2024-12-03T04:59:45,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0013f32453354c44949dd48ead1bc28d, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T04:59:45,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1fc1c9cce64fed6eabb56af3d145cb45 in 808ms, sequenceid=82, compaction requested=true 2024-12-03T04:59:45,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:45,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:59:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:59:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:59:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:45,073 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:59:45,073 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:59:45,079 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:59:45,081 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/B is initiating minor compaction (all files) 2024-12-03T04:59:45,082 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/B in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:45,082 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f0def0871b7443279c90cf0f9301acd1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/208193c6a60348659c942e58c96996b4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a48473c0360a4e09917f3aad40e74808, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/0ed2391fa37749d28452ef02bef5c28c] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=42.3 K 2024-12-03T04:59:45,084 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52676 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:59:45,084 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting f0def0871b7443279c90cf0f9301acd1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733201981754 2024-12-03T04:59:45,085 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 208193c6a60348659c942e58c96996b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733201982520 2024-12-03T04:59:45,089 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/A is initiating minor compaction (all files) 2024-12-03T04:59:45,089 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/A in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:45,090 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/3d7f0add986f4f69a33b4b6554c73172, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cfe53779d944f6abc414c01d9489682, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6b21e5248f6e499caf4ff6613f5af5ce, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/60df560727b24328824e916e59aba2d3] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=51.4 K 2024-12-03T04:59:45,092 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d7f0add986f4f69a33b4b6554c73172, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733201981722 2024-12-03T04:59:45,092 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cfe53779d944f6abc414c01d9489682, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733201982520 2024-12-03T04:59:45,093 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b21e5248f6e499caf4ff6613f5af5ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733201983267 2024-12-03T04:59:45,094 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60df560727b24328824e916e59aba2d3, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733201983538 2024-12-03T04:59:45,098 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a48473c0360a4e09917f3aad40e74808, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733201983274 2024-12-03T04:59:45,100 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ed2391fa37749d28452ef02bef5c28c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733201983559 2024-12-03T04:59:45,189 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#B#compaction#13 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:45,190 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#A#compaction#12 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:45,191 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/e4030eef25c848a98d98da964390508a is 50, key is test_row_0/B:col10/1733201984252/Put/seqid=0 2024-12-03T04:59:45,191 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/52fce1d7ae04486997afa447d86f59ae is 50, key is test_row_0/A:col10/1733201984252/Put/seqid=0 2024-12-03T04:59:45,219 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T04:59:45,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741849_1025 (size=12139) 2024-12-03T04:59:45,262 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/52fce1d7ae04486997afa447d86f59ae as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/52fce1d7ae04486997afa447d86f59ae 2024-12-03T04:59:45,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741850_1026 (size=12139) 2024-12-03T04:59:45,294 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/e4030eef25c848a98d98da964390508a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e4030eef25c848a98d98da964390508a 2024-12-03T04:59:45,307 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/A of 1fc1c9cce64fed6eabb56af3d145cb45 into 52fce1d7ae04486997afa447d86f59ae(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:45,307 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:45,310 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/A, priority=12, startTime=1733201985068; duration=0sec 2024-12-03T04:59:45,311 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:45,311 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:A 2024-12-03T04:59:45,311 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:59:45,332 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/B of 1fc1c9cce64fed6eabb56af3d145cb45 into e4030eef25c848a98d98da964390508a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:45,332 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:45,332 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/B, priority=12, startTime=1733201985072; duration=0sec 2024-12-03T04:59:45,332 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:45,333 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:B 2024-12-03T04:59:45,351 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T04:59:45,361 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:59:45,361 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/C is initiating minor compaction (all files) 2024-12-03T04:59:45,361 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/C in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:45,361 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/5f08f6f99b7d448ab8709e186d09d5bf, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/ef6e7c3c5b5747dda3e17ef837158694, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/e751cd436fee4403b5a18168d1cb8700, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0013f32453354c44949dd48ead1bc28d] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=42.3 K 2024-12-03T04:59:45,362 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f08f6f99b7d448ab8709e186d09d5bf, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733201981754 2024-12-03T04:59:45,363 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef6e7c3c5b5747dda3e17ef837158694, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733201982520 2024-12-03T04:59:45,364 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e751cd436fee4403b5a18168d1cb8700, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733201983274 2024-12-03T04:59:45,365 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0013f32453354c44949dd48ead1bc28d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733201983559 2024-12-03T04:59:45,409 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#C#compaction#14 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:45,410 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/9a5a038d305f4f87acd61f58e4e9b2ae is 50, key is test_row_0/C:col10/1733201984252/Put/seqid=0 2024-12-03T04:59:45,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741851_1027 (size=12139) 2024-12-03T04:59:45,462 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/9a5a038d305f4f87acd61f58e4e9b2ae as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/9a5a038d305f4f87acd61f58e4e9b2ae 2024-12-03T04:59:45,479 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:45,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T04:59:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:45,486 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/C of 1fc1c9cce64fed6eabb56af3d145cb45 into 9a5a038d305f4f87acd61f58e4e9b2ae(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:45,486 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:45,486 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/C, priority=12, startTime=1733201985072; duration=0sec 2024-12-03T04:59:45,486 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:45,486 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:C 2024-12-03T04:59:45,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/2ee0806acebf4cbf827e988ef8ee11cd is 50, key is test_row_0/A:col10/1733201985470/Put/seqid=0 2024-12-03T04:59:45,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741852_1028 (size=14341) 2024-12-03T04:59:45,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/2ee0806acebf4cbf827e988ef8ee11cd 2024-12-03T04:59:45,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/76145898de99419b83338e33afacf327 is 50, key is test_row_0/B:col10/1733201985470/Put/seqid=0 2024-12-03T04:59:45,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202045578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202045609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202045610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202045610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202045613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741853_1029 (size=12001) 2024-12-03T04:59:45,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/76145898de99419b83338e33afacf327 2024-12-03T04:59:45,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/51e7f76fa5d740fb92f8098cddca2786 is 50, key is test_row_0/C:col10/1733201985470/Put/seqid=0 2024-12-03T04:59:45,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202045712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741854_1030 (size=12001) 2024-12-03T04:59:45,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/51e7f76fa5d740fb92f8098cddca2786 2024-12-03T04:59:45,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202045722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202045722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202045722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202045722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/2ee0806acebf4cbf827e988ef8ee11cd as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/2ee0806acebf4cbf827e988ef8ee11cd 2024-12-03T04:59:45,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/2ee0806acebf4cbf827e988ef8ee11cd, entries=200, sequenceid=95, filesize=14.0 K 2024-12-03T04:59:45,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/76145898de99419b83338e33afacf327 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/76145898de99419b83338e33afacf327 2024-12-03T04:59:45,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/76145898de99419b83338e33afacf327, entries=150, sequenceid=95, filesize=11.7 K 2024-12-03T04:59:45,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/51e7f76fa5d740fb92f8098cddca2786 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/51e7f76fa5d740fb92f8098cddca2786 2024-12-03T04:59:45,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/51e7f76fa5d740fb92f8098cddca2786, entries=150, sequenceid=95, filesize=11.7 K 2024-12-03T04:59:45,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1fc1c9cce64fed6eabb56af3d145cb45 in 318ms, sequenceid=95, compaction requested=false 2024-12-03T04:59:45,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:59:45,811 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:59:45,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:59:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-03T04:59:45,815 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:59:45,817 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:59:45,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:59:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T04:59:45,926 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:45,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-03T04:59:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T04:59:45,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/80e93d06b71742acbad42b17d5d36cf5 is 50, key is test_row_0/A:col10/1733201985590/Put/seqid=0 2024-12-03T04:59:45,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202045957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202045958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202045953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T04:59:45,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202045967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:45,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:45,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:45,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:45,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202045970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:45,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741855_1031 (size=14341) 2024-12-03T04:59:45,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/80e93d06b71742acbad42b17d5d36cf5 2024-12-03T04:59:46,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/d32fb950059646f2a023b94fc388b7e7 is 50, key is test_row_0/B:col10/1733201985590/Put/seqid=0 2024-12-03T04:59:46,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741856_1032 (size=12001) 2024-12-03T04:59:46,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/d32fb950059646f2a023b94fc388b7e7 2024-12-03T04:59:46,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202046081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202046081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202046082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/0bb4b78f84da4cbea7344c38ccbf08a6 is 50, key is test_row_0/C:col10/1733201985590/Put/seqid=0 2024-12-03T04:59:46,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202046091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741857_1033 (size=12001) 2024-12-03T04:59:46,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/0bb4b78f84da4cbea7344c38ccbf08a6 2024-12-03T04:59:46,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202046091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/80e93d06b71742acbad42b17d5d36cf5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/80e93d06b71742acbad42b17d5d36cf5 2024-12-03T04:59:46,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/80e93d06b71742acbad42b17d5d36cf5, entries=200, sequenceid=123, filesize=14.0 K 2024-12-03T04:59:46,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T04:59:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:46,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:46,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T04:59:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:46,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/d32fb950059646f2a023b94fc388b7e7 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d32fb950059646f2a023b94fc388b7e7 2024-12-03T04:59:46,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d32fb950059646f2a023b94fc388b7e7, entries=150, sequenceid=123, filesize=11.7 K 2024-12-03T04:59:46,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/0bb4b78f84da4cbea7344c38ccbf08a6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0bb4b78f84da4cbea7344c38ccbf08a6 2024-12-03T04:59:46,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0bb4b78f84da4cbea7344c38ccbf08a6, entries=150, sequenceid=123, filesize=11.7 K 2024-12-03T04:59:46,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 1fc1c9cce64fed6eabb56af3d145cb45 in 262ms, sequenceid=123, compaction requested=true 2024-12-03T04:59:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:59:46,189 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:59:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:59:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:46,189 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:46,193 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:46,193 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/B is initiating minor compaction (all files) 2024-12-03T04:59:46,193 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/B in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:46,193 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:46,193 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e4030eef25c848a98d98da964390508a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/76145898de99419b83338e33afacf327, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d32fb950059646f2a023b94fc388b7e7] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=35.3 K 2024-12-03T04:59:46,193 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/A is initiating minor compaction (all files) 2024-12-03T04:59:46,194 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/A in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:46,194 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/52fce1d7ae04486997afa447d86f59ae, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/2ee0806acebf4cbf827e988ef8ee11cd, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/80e93d06b71742acbad42b17d5d36cf5] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=39.9 K 2024-12-03T04:59:46,195 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting e4030eef25c848a98d98da964390508a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733201983559 2024-12-03T04:59:46,195 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52fce1d7ae04486997afa447d86f59ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733201983559 2024-12-03T04:59:46,196 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ee0806acebf4cbf827e988ef8ee11cd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733201984269 2024-12-03T04:59:46,197 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 76145898de99419b83338e33afacf327, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733201984269 2024-12-03T04:59:46,197 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80e93d06b71742acbad42b17d5d36cf5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733201985576 2024-12-03T04:59:46,198 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting d32fb950059646f2a023b94fc388b7e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733201985576 2024-12-03T04:59:46,231 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:46,232 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/6b188b3e5a0946b88944c225550f7f03 is 50, key is test_row_0/B:col10/1733201985590/Put/seqid=0 2024-12-03T04:59:46,233 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:46,234 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/2d6af0a72118497099eee4c18ce66aed is 50, key is test_row_0/A:col10/1733201985590/Put/seqid=0 2024-12-03T04:59:46,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741858_1034 (size=12241) 2024-12-03T04:59:46,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T04:59:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:46,295 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-03T04:59:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:46,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:46,306 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/6b188b3e5a0946b88944c225550f7f03 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/6b188b3e5a0946b88944c225550f7f03 2024-12-03T04:59:46,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741859_1035 (size=12241) 2024-12-03T04:59:46,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/8cf04ff5ec89458faac67521bbf1c92b is 50, key is test_row_0/A:col10/1733201985963/Put/seqid=0 2024-12-03T04:59:46,346 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/B of 1fc1c9cce64fed6eabb56af3d145cb45 into 6b188b3e5a0946b88944c225550f7f03(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:46,346 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/2d6af0a72118497099eee4c18ce66aed as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/2d6af0a72118497099eee4c18ce66aed 2024-12-03T04:59:46,347 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:46,347 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/B, priority=13, startTime=1733201986189; duration=0sec 2024-12-03T04:59:46,347 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:46,347 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:B 2024-12-03T04:59:46,347 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:46,351 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:46,351 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/C is initiating minor compaction (all files) 2024-12-03T04:59:46,351 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/C in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:46,351 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/9a5a038d305f4f87acd61f58e4e9b2ae, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/51e7f76fa5d740fb92f8098cddca2786, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0bb4b78f84da4cbea7344c38ccbf08a6] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=35.3 K 2024-12-03T04:59:46,353 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a5a038d305f4f87acd61f58e4e9b2ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733201983559 2024-12-03T04:59:46,362 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 51e7f76fa5d740fb92f8098cddca2786, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733201984269 2024-12-03T04:59:46,365 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb4b78f84da4cbea7344c38ccbf08a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733201985576 2024-12-03T04:59:46,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741860_1036 (size=12151) 2024-12-03T04:59:46,369 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/8cf04ff5ec89458faac67521bbf1c92b 2024-12-03T04:59:46,385 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/A of 1fc1c9cce64fed6eabb56af3d145cb45 into 2d6af0a72118497099eee4c18ce66aed(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:46,385 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:46,385 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/A, priority=13, startTime=1733201986189; duration=0sec 2024-12-03T04:59:46,386 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:46,386 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:A 2024-12-03T04:59:46,395 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#C#compaction#24 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:46,396 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/930f2bdf7b5f45df89d82dd6f23cf515 is 50, key is test_row_0/C:col10/1733201985590/Put/seqid=0 2024-12-03T04:59:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/e4c3d2f401804c84a8b8e95d56804007 is 50, key is test_row_0/B:col10/1733201985963/Put/seqid=0 2024-12-03T04:59:46,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741861_1037 (size=12241) 2024-12-03T04:59:46,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202046386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202046388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202046390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202046394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202046397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,416 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/930f2bdf7b5f45df89d82dd6f23cf515 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/930f2bdf7b5f45df89d82dd6f23cf515 2024-12-03T04:59:46,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741862_1038 (size=12151) 2024-12-03T04:59:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,421 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/e4c3d2f401804c84a8b8e95d56804007 2024-12-03T04:59:46,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,446 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/C of 1fc1c9cce64fed6eabb56af3d145cb45 into 930f2bdf7b5f45df89d82dd6f23cf515(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:46,446 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:46,446 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/C, priority=13, startTime=1733201986189; duration=0sec 2024-12-03T04:59:46,447 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:46,447 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:C 2024-12-03T04:59:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T04:59:46,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/756243fb555d43ac83790792da083105 is 50, key is test_row_0/C:col10/1733201985963/Put/seqid=0 2024-12-03T04:59:46,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741863_1039 (size=12151) 2024-12-03T04:59:46,502 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/756243fb555d43ac83790792da083105 2024-12-03T04:59:46,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/8cf04ff5ec89458faac67521bbf1c92b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cf04ff5ec89458faac67521bbf1c92b 2024-12-03T04:59:46,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202046521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202046526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202046526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,531 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cf04ff5ec89458faac67521bbf1c92b, entries=150, sequenceid=133, filesize=11.9 K 2024-12-03T04:59:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202046526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202046532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/e4c3d2f401804c84a8b8e95d56804007 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e4c3d2f401804c84a8b8e95d56804007 2024-12-03T04:59:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,543 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e4c3d2f401804c84a8b8e95d56804007, entries=150, sequenceid=133, filesize=11.9 K 2024-12-03T04:59:46,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/756243fb555d43ac83790792da083105 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/756243fb555d43ac83790792da083105 2024-12-03T04:59:46,557 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/756243fb555d43ac83790792da083105, entries=150, sequenceid=133, filesize=11.9 K 2024-12-03T04:59:46,566 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 1fc1c9cce64fed6eabb56af3d145cb45 in 271ms, sequenceid=133, compaction requested=false 2024-12-03T04:59:46,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:46,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:46,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-03T04:59:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-03T04:59:46,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-03T04:59:46,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 752 msec 2024-12-03T04:59:46,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 762 msec 2024-12-03T04:59:46,690 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T04:59:46,691 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T04:59:46,695 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T04:59:46,695 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T04:59:46,696 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T04:59:46,696 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T04:59:46,706 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T04:59:46,706 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T04:59:46,749 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:46,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-03T04:59:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:46,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202046757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/ab507cd63ed04e15a896704d42370693 is 50, key is test_row_0/A:col10/1733201986378/Put/seqid=0 2024-12-03T04:59:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202046759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202046763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202046767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202046770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741864_1040 (size=16931) 2024-12-03T04:59:46,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202046881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202046886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202046891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202046891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202046891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T04:59:46,966 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:59:46,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:59:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-03T04:59:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:59:46,990 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:59:46,991 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:59:46,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:59:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:59:47,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202047101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202047100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202047129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202047129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202047130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:59:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/ab507cd63ed04e15a896704d42370693 2024-12-03T04:59:47,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/a8dfc54b850945dba325072f77f5caa2 is 50, key is test_row_0/B:col10/1733201986378/Put/seqid=0 2024-12-03T04:59:47,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741865_1041 (size=12151) 2024-12-03T04:59:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:59:47,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:59:47,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:47,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202047441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202047440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202047441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202047451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202047452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:59:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:59:47,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:59:47,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/a8dfc54b850945dba325072f77f5caa2 2024-12-03T04:59:47,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/4a750660366f415686db1e695d012850 is 50, key is test_row_0/C:col10/1733201986378/Put/seqid=0 2024-12-03T04:59:47,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741866_1042 (size=12151) 2024-12-03T04:59:47,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:59:47,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:47,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:59:47,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:47,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:47,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:47,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202047950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202047951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202047951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202047961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202047962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:48,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:59:48,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:48,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:48,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:48,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:48,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:59:48,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/4a750660366f415686db1e695d012850 2024-12-03T04:59:48,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/ab507cd63ed04e15a896704d42370693 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ab507cd63ed04e15a896704d42370693 2024-12-03T04:59:48,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ab507cd63ed04e15a896704d42370693, entries=250, sequenceid=164, filesize=16.5 K 2024-12-03T04:59:48,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/a8dfc54b850945dba325072f77f5caa2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a8dfc54b850945dba325072f77f5caa2 2024-12-03T04:59:48,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a8dfc54b850945dba325072f77f5caa2, entries=150, sequenceid=164, filesize=11.9 K 2024-12-03T04:59:48,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/4a750660366f415686db1e695d012850 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/4a750660366f415686db1e695d012850 2024-12-03T04:59:48,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/4a750660366f415686db1e695d012850, entries=150, sequenceid=164, filesize=11.9 K 2024-12-03T04:59:48,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 1fc1c9cce64fed6eabb56af3d145cb45 in 1432ms, sequenceid=164, compaction requested=true 2024-12-03T04:59:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:59:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:48,182 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:59:48,182 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:59:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:48,184 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41323 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:48,184 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/A is initiating minor compaction (all files) 2024-12-03T04:59:48,184 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/A in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:48,185 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/2d6af0a72118497099eee4c18ce66aed, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cf04ff5ec89458faac67521bbf1c92b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ab507cd63ed04e15a896704d42370693] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=40.4 K 2024-12-03T04:59:48,186 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:48,186 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/B is initiating minor compaction (all files) 2024-12-03T04:59:48,186 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/B in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:48,186 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/6b188b3e5a0946b88944c225550f7f03, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e4c3d2f401804c84a8b8e95d56804007, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a8dfc54b850945dba325072f77f5caa2] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=35.7 K 2024-12-03T04:59:48,186 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d6af0a72118497099eee4c18ce66aed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733201985576 2024-12-03T04:59:48,187 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b188b3e5a0946b88944c225550f7f03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733201985576 2024-12-03T04:59:48,190 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cf04ff5ec89458faac67521bbf1c92b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733201985952 2024-12-03T04:59:48,190 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4c3d2f401804c84a8b8e95d56804007, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733201985952 2024-12-03T04:59:48,191 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ab507cd63ed04e15a896704d42370693, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733201986341 2024-12-03T04:59:48,191 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8dfc54b850945dba325072f77f5caa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733201986378 2024-12-03T04:59:48,211 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:48,213 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/7874ba34a55c496987e7754b784a0d57 is 50, key is test_row_0/A:col10/1733201986378/Put/seqid=0 2024-12-03T04:59:48,223 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#B#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:48,224 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/f79d79aa2fe94d0e88ea3fdcbe84c8ef is 50, key is test_row_0/B:col10/1733201986378/Put/seqid=0 2024-12-03T04:59:48,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741867_1043 (size=12493) 2024-12-03T04:59:48,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741868_1044 (size=12493) 2024-12-03T04:59:48,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:59:48,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:48,244 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-03T04:59:48,251 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/f79d79aa2fe94d0e88ea3fdcbe84c8ef as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f79d79aa2fe94d0e88ea3fdcbe84c8ef 2024-12-03T04:59:48,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/4d3b43e664864dd19ec3f24dc2665201 is 50, key is test_row_0/A:col10/1733201986764/Put/seqid=0 2024-12-03T04:59:48,263 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/B of 1fc1c9cce64fed6eabb56af3d145cb45 into f79d79aa2fe94d0e88ea3fdcbe84c8ef(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:48,263 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:48,264 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/B, priority=13, startTime=1733201988182; duration=0sec 2024-12-03T04:59:48,264 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:48,264 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:B 2024-12-03T04:59:48,264 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:48,266 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:48,266 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/C is initiating minor compaction (all files) 2024-12-03T04:59:48,266 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/C in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:48,266 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/930f2bdf7b5f45df89d82dd6f23cf515, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/756243fb555d43ac83790792da083105, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/4a750660366f415686db1e695d012850] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=35.7 K 2024-12-03T04:59:48,267 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 930f2bdf7b5f45df89d82dd6f23cf515, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733201985576 2024-12-03T04:59:48,268 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 756243fb555d43ac83790792da083105, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733201985952 2024-12-03T04:59:48,268 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a750660366f415686db1e695d012850, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733201986378 2024-12-03T04:59:48,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741869_1045 (size=12151) 2024-12-03T04:59:48,289 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#C#compaction#33 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:48,291 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/621316aff8134347ba541161937c5be1 is 50, key is test_row_0/C:col10/1733201986378/Put/seqid=0 2024-12-03T04:59:48,298 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/4d3b43e664864dd19ec3f24dc2665201 2024-12-03T04:59:48,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741870_1046 (size=12493) 2024-12-03T04:59:48,318 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/621316aff8134347ba541161937c5be1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/621316aff8134347ba541161937c5be1 2024-12-03T04:59:48,329 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/C of 1fc1c9cce64fed6eabb56af3d145cb45 into 621316aff8134347ba541161937c5be1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:48,329 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:48,329 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/C, priority=13, startTime=1733201988182; duration=0sec 2024-12-03T04:59:48,329 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:48,329 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:C 2024-12-03T04:59:48,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/e5190e3f90da4107bca15862a16a7d64 is 50, key is test_row_0/B:col10/1733201986764/Put/seqid=0 2024-12-03T04:59:48,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741871_1047 (size=12151) 2024-12-03T04:59:48,354 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/e5190e3f90da4107bca15862a16a7d64 2024-12-03T04:59:48,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/969babe6b1534388bf89199109158ded is 50, key is test_row_0/C:col10/1733201986764/Put/seqid=0 2024-12-03T04:59:48,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741872_1048 (size=12151) 2024-12-03T04:59:48,392 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/969babe6b1534388bf89199109158ded 2024-12-03T04:59:48,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/4d3b43e664864dd19ec3f24dc2665201 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/4d3b43e664864dd19ec3f24dc2665201 2024-12-03T04:59:48,415 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/4d3b43e664864dd19ec3f24dc2665201, entries=150, sequenceid=173, filesize=11.9 K 2024-12-03T04:59:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/e5190e3f90da4107bca15862a16a7d64 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e5190e3f90da4107bca15862a16a7d64 2024-12-03T04:59:48,432 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e5190e3f90da4107bca15862a16a7d64, entries=150, sequenceid=173, filesize=11.9 K 2024-12-03T04:59:48,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/969babe6b1534388bf89199109158ded as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/969babe6b1534388bf89199109158ded 2024-12-03T04:59:48,450 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/969babe6b1534388bf89199109158ded, entries=150, sequenceid=173, filesize=11.9 K 2024-12-03T04:59:48,452 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 1fc1c9cce64fed6eabb56af3d145cb45 in 208ms, sequenceid=173, compaction requested=false 2024-12-03T04:59:48,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:48,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:48,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-03T04:59:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-03T04:59:48,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T04:59:48,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4640 sec 2024-12-03T04:59:48,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4870 sec 2024-12-03T04:59:48,638 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/7874ba34a55c496987e7754b784a0d57 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7874ba34a55c496987e7754b784a0d57 2024-12-03T04:59:48,650 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/A of 1fc1c9cce64fed6eabb56af3d145cb45 into 7874ba34a55c496987e7754b784a0d57(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:48,650 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:48,650 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/A, priority=13, startTime=1733201988182; duration=0sec 2024-12-03T04:59:48,651 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:48,651 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:A 2024-12-03T04:59:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:48,981 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:48,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T04:59:49,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/d7b43a6ec4694494a55471e537e9c658 is 50, key is test_row_0/A:col10/1733201988979/Put/seqid=0 2024-12-03T04:59:49,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741873_1049 (size=14541) 2024-12-03T04:59:49,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/d7b43a6ec4694494a55471e537e9c658 2024-12-03T04:59:49,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/678ca55d873a46578f77869fe3c1f5e0 is 50, key is test_row_0/B:col10/1733201988979/Put/seqid=0 2024-12-03T04:59:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202049039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202049043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202049043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202049051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741874_1050 (size=12151) 2024-12-03T04:59:49,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/678ca55d873a46578f77869fe3c1f5e0 2024-12-03T04:59:49,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202049065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/42470e497df14ba0afbe258bb881c1d0 is 50, key is test_row_0/C:col10/1733201988979/Put/seqid=0 2024-12-03T04:59:49,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741875_1051 (size=12151) 2024-12-03T04:59:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:59:49,132 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:59:49,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:59:49,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-03T04:59:49,135 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:59:49,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:59:49,137 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:59:49,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:59:49,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202049160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202049161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202049161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202049170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202049191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:59:49,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T04:59:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:49,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:49,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:49,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202049371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202049371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202049371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202049381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202049401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T04:59:49,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:49,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:59:49,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/42470e497df14ba0afbe258bb881c1d0 2024-12-03T04:59:49,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/d7b43a6ec4694494a55471e537e9c658 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d7b43a6ec4694494a55471e537e9c658 2024-12-03T04:59:49,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d7b43a6ec4694494a55471e537e9c658, entries=200, sequenceid=187, filesize=14.2 K 2024-12-03T04:59:49,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/678ca55d873a46578f77869fe3c1f5e0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/678ca55d873a46578f77869fe3c1f5e0 2024-12-03T04:59:49,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/678ca55d873a46578f77869fe3c1f5e0, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T04:59:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/42470e497df14ba0afbe258bb881c1d0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/42470e497df14ba0afbe258bb881c1d0 2024-12-03T04:59:49,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/42470e497df14ba0afbe258bb881c1d0, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T04:59:49,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1fc1c9cce64fed6eabb56af3d145cb45 in 603ms, sequenceid=187, compaction requested=true 2024-12-03T04:59:49,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:49,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:59:49,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:49,585 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:49,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:59:49,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:49,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:59:49,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:49,585 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:49,586 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:49,586 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/A is initiating minor compaction (all files) 2024-12-03T04:59:49,587 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:49,587 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/A in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:49,587 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/B is initiating minor compaction (all files) 2024-12-03T04:59:49,587 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/B in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:49,587 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7874ba34a55c496987e7754b784a0d57, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/4d3b43e664864dd19ec3f24dc2665201, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d7b43a6ec4694494a55471e537e9c658] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=38.3 K 2024-12-03T04:59:49,587 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f79d79aa2fe94d0e88ea3fdcbe84c8ef, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e5190e3f90da4107bca15862a16a7d64, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/678ca55d873a46578f77869fe3c1f5e0] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=35.9 K 2024-12-03T04:59:49,588 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting f79d79aa2fe94d0e88ea3fdcbe84c8ef, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733201986378 2024-12-03T04:59:49,588 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7874ba34a55c496987e7754b784a0d57, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733201986378 2024-12-03T04:59:49,588 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting e5190e3f90da4107bca15862a16a7d64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733201986760 2024-12-03T04:59:49,588 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d3b43e664864dd19ec3f24dc2665201, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733201986760 2024-12-03T04:59:49,589 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 678ca55d873a46578f77869fe3c1f5e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733201988970 2024-12-03T04:59:49,589 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7b43a6ec4694494a55471e537e9c658, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733201988970 2024-12-03T04:59:49,604 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:49,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T04:59:49,605 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/c7313a8f9b5b486aac20b4955303c80b is 50, key is test_row_0/B:col10/1733201988979/Put/seqid=0 2024-12-03T04:59:49,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:49,605 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-03T04:59:49,614 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#A#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:49,615 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/002c9e44bb7e478e85a491c9a85a10d3 is 50, key is test_row_0/A:col10/1733201988979/Put/seqid=0 2024-12-03T04:59:49,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/5185627d54f54714acfeb43bf5f57818 is 50, key is test_row_0/A:col10/1733201989044/Put/seqid=0 2024-12-03T04:59:49,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741876_1052 (size=12595) 2024-12-03T04:59:49,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741877_1053 (size=12595) 2024-12-03T04:59:49,678 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/c7313a8f9b5b486aac20b4955303c80b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c7313a8f9b5b486aac20b4955303c80b 2024-12-03T04:59:49,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:49,690 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/B of 1fc1c9cce64fed6eabb56af3d145cb45 into c7313a8f9b5b486aac20b4955303c80b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:49,690 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:49,690 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/B, priority=13, startTime=1733201989585; duration=0sec 2024-12-03T04:59:49,690 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:49,690 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:B 2024-12-03T04:59:49,690 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:49,692 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:49,692 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/C is initiating minor compaction (all files) 2024-12-03T04:59:49,692 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/C in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:49,692 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/621316aff8134347ba541161937c5be1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/969babe6b1534388bf89199109158ded, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/42470e497df14ba0afbe258bb881c1d0] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=35.9 K 2024-12-03T04:59:49,693 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 621316aff8134347ba541161937c5be1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733201986378 2024-12-03T04:59:49,694 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 969babe6b1534388bf89199109158ded, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733201986760 2024-12-03T04:59:49,695 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 42470e497df14ba0afbe258bb881c1d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733201988970 2024-12-03T04:59:49,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741878_1054 (size=12151) 2024-12-03T04:59:49,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,707 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/5185627d54f54714acfeb43bf5f57818 2024-12-03T04:59:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202049696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202049701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202049704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,724 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:49,725 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/79b4a16e91114352a8ff90aa6e6346e4 is 50, key is test_row_0/C:col10/1733201988979/Put/seqid=0 2024-12-03T04:59:49,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202049707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202049720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/f681de595bc94c4ba0fae59d33b788e5 is 50, key is test_row_0/B:col10/1733201989044/Put/seqid=0 2024-12-03T04:59:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:59:49,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741879_1055 (size=12595) 2024-12-03T04:59:49,780 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/79b4a16e91114352a8ff90aa6e6346e4 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/79b4a16e91114352a8ff90aa6e6346e4 2024-12-03T04:59:49,789 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/C of 1fc1c9cce64fed6eabb56af3d145cb45 into 79b4a16e91114352a8ff90aa6e6346e4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:49,789 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:49,789 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/C, priority=13, startTime=1733201989585; duration=0sec 2024-12-03T04:59:49,789 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:49,790 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:C 2024-12-03T04:59:49,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741880_1056 (size=12151) 2024-12-03T04:59:49,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202049810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202049821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202049823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:49,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202049831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202050025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202050031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202050031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202050040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,079 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/002c9e44bb7e478e85a491c9a85a10d3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/002c9e44bb7e478e85a491c9a85a10d3 2024-12-03T04:59:50,101 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/A of 1fc1c9cce64fed6eabb56af3d145cb45 into 002c9e44bb7e478e85a491c9a85a10d3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:50,101 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:50,101 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/A, priority=13, startTime=1733201989584; duration=0sec 2024-12-03T04:59:50,101 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:50,101 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:A 2024-12-03T04:59:50,196 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/f681de595bc94c4ba0fae59d33b788e5 2024-12-03T04:59:50,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202050231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/1d55e16503be4fa2ba4c6d351073e94b is 50, key is test_row_0/C:col10/1733201989044/Put/seqid=0 2024-12-03T04:59:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:59:50,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741881_1057 (size=12151) 2024-12-03T04:59:50,272 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/1d55e16503be4fa2ba4c6d351073e94b 2024-12-03T04:59:50,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/5185627d54f54714acfeb43bf5f57818 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/5185627d54f54714acfeb43bf5f57818 2024-12-03T04:59:50,294 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/5185627d54f54714acfeb43bf5f57818, entries=150, sequenceid=213, filesize=11.9 K 2024-12-03T04:59:50,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/f681de595bc94c4ba0fae59d33b788e5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f681de595bc94c4ba0fae59d33b788e5 2024-12-03T04:59:50,310 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f681de595bc94c4ba0fae59d33b788e5, entries=150, sequenceid=213, filesize=11.9 K 2024-12-03T04:59:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/1d55e16503be4fa2ba4c6d351073e94b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1d55e16503be4fa2ba4c6d351073e94b 2024-12-03T04:59:50,325 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1d55e16503be4fa2ba4c6d351073e94b, entries=150, sequenceid=213, filesize=11.9 K 2024-12-03T04:59:50,326 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1fc1c9cce64fed6eabb56af3d145cb45 in 721ms, sequenceid=213, compaction requested=false 2024-12-03T04:59:50,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:50,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:50,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-03T04:59:50,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-03T04:59:50,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T04:59:50,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1910 sec 2024-12-03T04:59:50,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2020 sec 2024-12-03T04:59:50,350 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:50,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-03T04:59:50,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:50,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/033ddf637f89410da665bfbf6c69e04b is 50, key is test_row_0/A:col10/1733201989703/Put/seqid=0 2024-12-03T04:59:50,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741882_1058 (size=12151) 2024-12-03T04:59:50,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202050437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202050440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202050445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202050457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202050561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202050561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202050605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202050605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202050771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202050771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202050810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/033ddf637f89410da665bfbf6c69e04b 2024-12-03T04:59:50,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202050826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:50,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/d53405749e2740e6894fd41b963a121d is 50, key is test_row_0/B:col10/1733201989703/Put/seqid=0 2024-12-03T04:59:50,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741883_1059 (size=12151) 2024-12-03T04:59:50,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/d53405749e2740e6894fd41b963a121d 2024-12-03T04:59:50,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/2c166ed184884e8f9a0c689fb5e957ce is 50, key is test_row_0/C:col10/1733201989703/Put/seqid=0 2024-12-03T04:59:50,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741884_1060 (size=12151) 2024-12-03T04:59:50,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/2c166ed184884e8f9a0c689fb5e957ce 2024-12-03T04:59:50,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/033ddf637f89410da665bfbf6c69e04b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/033ddf637f89410da665bfbf6c69e04b 2024-12-03T04:59:50,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/033ddf637f89410da665bfbf6c69e04b, entries=150, sequenceid=229, filesize=11.9 K 2024-12-03T04:59:50,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/d53405749e2740e6894fd41b963a121d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d53405749e2740e6894fd41b963a121d 2024-12-03T04:59:50,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d53405749e2740e6894fd41b963a121d, entries=150, sequenceid=229, filesize=11.9 K 2024-12-03T04:59:50,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/2c166ed184884e8f9a0c689fb5e957ce as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2c166ed184884e8f9a0c689fb5e957ce 2024-12-03T04:59:50,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2c166ed184884e8f9a0c689fb5e957ce, entries=150, sequenceid=229, filesize=11.9 K 2024-12-03T04:59:50,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 1fc1c9cce64fed6eabb56af3d145cb45 in 648ms, sequenceid=229, compaction requested=true 2024-12-03T04:59:50,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:50,999 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:51,000 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:51,000 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/A is initiating minor compaction (all files) 2024-12-03T04:59:51,001 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/A in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:51,001 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/002c9e44bb7e478e85a491c9a85a10d3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/5185627d54f54714acfeb43bf5f57818, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/033ddf637f89410da665bfbf6c69e04b] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=36.0 K 2024-12-03T04:59:51,001 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 002c9e44bb7e478e85a491c9a85a10d3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733201988970 2024-12-03T04:59:51,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:59:51,002 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5185627d54f54714acfeb43bf5f57818, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733201989039 2024-12-03T04:59:51,002 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 033ddf637f89410da665bfbf6c69e04b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201989703 2024-12-03T04:59:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:59:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:51,011 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:59:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:51,012 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:51,012 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/B is initiating minor compaction (all files) 2024-12-03T04:59:51,012 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/B in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:51,012 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c7313a8f9b5b486aac20b4955303c80b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f681de595bc94c4ba0fae59d33b788e5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d53405749e2740e6894fd41b963a121d] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=36.0 K 2024-12-03T04:59:51,013 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c7313a8f9b5b486aac20b4955303c80b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733201988970 2024-12-03T04:59:51,013 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting f681de595bc94c4ba0fae59d33b788e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733201989039 2024-12-03T04:59:51,014 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting d53405749e2740e6894fd41b963a121d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201989703 2024-12-03T04:59:51,028 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#A#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:51,028 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/05a0930c9a00455199dfc6f9df82d6a6 is 50, key is test_row_0/A:col10/1733201989703/Put/seqid=0 2024-12-03T04:59:51,042 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:51,042 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/9b39e91e24f74b51b4ebba013a464059 is 50, key is test_row_0/B:col10/1733201989703/Put/seqid=0 2024-12-03T04:59:51,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741885_1061 (size=12697) 2024-12-03T04:59:51,074 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/05a0930c9a00455199dfc6f9df82d6a6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/05a0930c9a00455199dfc6f9df82d6a6 2024-12-03T04:59:51,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741886_1062 (size=12697) 2024-12-03T04:59:51,087 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/A of 1fc1c9cce64fed6eabb56af3d145cb45 into 05a0930c9a00455199dfc6f9df82d6a6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:51,087 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:51,087 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/A, priority=13, startTime=1733201990999; duration=0sec 2024-12-03T04:59:51,088 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:51,088 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:A 2024-12-03T04:59:51,089 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:51,090 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/9b39e91e24f74b51b4ebba013a464059 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/9b39e91e24f74b51b4ebba013a464059 2024-12-03T04:59:51,090 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:51,091 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/C is initiating minor compaction (all files) 2024-12-03T04:59:51,091 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/C in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:51,091 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/79b4a16e91114352a8ff90aa6e6346e4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1d55e16503be4fa2ba4c6d351073e94b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2c166ed184884e8f9a0c689fb5e957ce] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=36.0 K 2024-12-03T04:59:51,092 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b4a16e91114352a8ff90aa6e6346e4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733201988970 2024-12-03T04:59:51,092 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d55e16503be4fa2ba4c6d351073e94b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733201989039 2024-12-03T04:59:51,093 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c166ed184884e8f9a0c689fb5e957ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201989703 2024-12-03T04:59:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:51,096 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:51,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-03T04:59:51,099 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/B of 1fc1c9cce64fed6eabb56af3d145cb45 into 9b39e91e24f74b51b4ebba013a464059(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:51,099 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:51,100 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/B, priority=13, startTime=1733201991011; duration=0sec 2024-12-03T04:59:51,100 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:51,100 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:B 2024-12-03T04:59:51,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/7c5dba6f424048ce9419f665246fe165 is 50, key is test_row_0/A:col10/1733201991085/Put/seqid=0 2024-12-03T04:59:51,123 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#C#compaction#51 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:51,124 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/94d09ec6438241e787c192731266afd5 is 50, key is test_row_0/C:col10/1733201989703/Put/seqid=0 2024-12-03T04:59:51,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202051127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202051131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202051135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741887_1063 (size=16931) 2024-12-03T04:59:51,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202051140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741888_1064 (size=12697) 2024-12-03T04:59:51,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/7c5dba6f424048ce9419f665246fe165 2024-12-03T04:59:51,167 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/94d09ec6438241e787c192731266afd5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/94d09ec6438241e787c192731266afd5 2024-12-03T04:59:51,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/05cac77ed7264ebeacb9dcc0b2ab4094 is 50, key is test_row_0/B:col10/1733201991085/Put/seqid=0 2024-12-03T04:59:51,182 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/C of 1fc1c9cce64fed6eabb56af3d145cb45 into 94d09ec6438241e787c192731266afd5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:51,182 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:51,182 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/C, priority=13, startTime=1733201991011; duration=0sec 2024-12-03T04:59:51,183 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:51,183 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:C 2024-12-03T04:59:51,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741889_1065 (size=12151) 2024-12-03T04:59:51,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/05cac77ed7264ebeacb9dcc0b2ab4094 2024-12-03T04:59:51,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/1647dc26dfa14007ba1dfd823823fd36 is 50, key is test_row_0/C:col10/1733201991085/Put/seqid=0 2024-12-03T04:59:51,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202051240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202051241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,249 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in 1fc1c9cce64fed6eabb56af3d145cb45 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:59:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202051251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741890_1066 (size=12151) 2024-12-03T04:59:51,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/1647dc26dfa14007ba1dfd823823fd36 2024-12-03T04:59:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:59:51,281 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:59:51,283 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:59:51,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/7c5dba6f424048ce9419f665246fe165 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7c5dba6f424048ce9419f665246fe165 2024-12-03T04:59:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-03T04:59:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:59:51,294 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:59:51,295 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:59:51,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:59:51,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7c5dba6f424048ce9419f665246fe165, entries=250, sequenceid=256, filesize=16.5 K 2024-12-03T04:59:51,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/05cac77ed7264ebeacb9dcc0b2ab4094 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/05cac77ed7264ebeacb9dcc0b2ab4094 2024-12-03T04:59:51,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/05cac77ed7264ebeacb9dcc0b2ab4094, entries=150, sequenceid=256, filesize=11.9 K 2024-12-03T04:59:51,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/1647dc26dfa14007ba1dfd823823fd36 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1647dc26dfa14007ba1dfd823823fd36 2024-12-03T04:59:51,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1647dc26dfa14007ba1dfd823823fd36, entries=150, sequenceid=256, filesize=11.9 K 2024-12-03T04:59:51,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1fc1c9cce64fed6eabb56af3d145cb45 in 231ms, sequenceid=256, compaction requested=false 2024-12-03T04:59:51,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:59:51,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:59:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:51,452 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-03T04:59:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:51,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/6447ca6c63c24d969f061661ec5cabbc is 50, key is test_row_0/A:col10/1733201991452/Put/seqid=0 2024-12-03T04:59:51,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741891_1067 (size=12301) 2024-12-03T04:59:51,489 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/6447ca6c63c24d969f061661ec5cabbc 2024-12-03T04:59:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/5bf923e076fe47b2b4066f93c5df9514 is 50, key is test_row_0/B:col10/1733201991452/Put/seqid=0 2024-12-03T04:59:51,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202051529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202051532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741892_1068 (size=12301) 2024-12-03T04:59:51,561 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/5bf923e076fe47b2b4066f93c5df9514 2024-12-03T04:59:51,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/6899fdf3eb93487c8804e14751968515 is 50, key is test_row_0/C:col10/1733201991452/Put/seqid=0 2024-12-03T04:59:51,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741893_1069 (size=12301) 2024-12-03T04:59:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:59:51,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202051641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202051641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202051641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202051652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202051850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202051850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:59:51,998 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/6899fdf3eb93487c8804e14751968515 2024-12-03T04:59:52,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/6447ca6c63c24d969f061661ec5cabbc as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6447ca6c63c24d969f061661ec5cabbc 2024-12-03T04:59:52,024 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6447ca6c63c24d969f061661ec5cabbc, entries=150, sequenceid=269, filesize=12.0 K 2024-12-03T04:59:52,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/5bf923e076fe47b2b4066f93c5df9514 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5bf923e076fe47b2b4066f93c5df9514 2024-12-03T04:59:52,033 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5bf923e076fe47b2b4066f93c5df9514, entries=150, sequenceid=269, filesize=12.0 K 2024-12-03T04:59:52,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/6899fdf3eb93487c8804e14751968515 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/6899fdf3eb93487c8804e14751968515 2024-12-03T04:59:52,045 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/6899fdf3eb93487c8804e14751968515, entries=150, sequenceid=269, filesize=12.0 K 2024-12-03T04:59:52,048 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1fc1c9cce64fed6eabb56af3d145cb45 in 596ms, sequenceid=269, compaction requested=true 2024-12-03T04:59:52,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:52,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:52,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-03T04:59:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-03T04:59:52,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-03T04:59:52,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 756 msec 2024-12-03T04:59:52,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 772 msec 2024-12-03T04:59:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:52,165 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:52,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-03T04:59:52,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/d9e086aa8c18441a9843bcd091828f8a is 50, key is test_row_0/A:col10/1733201992164/Put/seqid=0 2024-12-03T04:59:52,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202052183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:52,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202052185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:52,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741894_1070 (size=14741) 2024-12-03T04:59:52,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/d9e086aa8c18441a9843bcd091828f8a 2024-12-03T04:59:52,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/84eccff191aa4cfe82f8ea2c1bdc0301 is 50, key is test_row_0/B:col10/1733201992164/Put/seqid=0 2024-12-03T04:59:52,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741895_1071 (size=12301) 2024-12-03T04:59:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202052300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:52,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202052301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:59:52,435 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:59:52,435 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:59:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-03T04:59:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:59:52,439 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:59:52,440 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:59:52,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:59:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202052511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:52,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202052512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:59:52,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:59:52,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:52,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:52,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:52,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:52,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:52,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/84eccff191aa4cfe82f8ea2c1bdc0301 2024-12-03T04:59:52,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202052661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202052661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:52,665 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in 1fc1c9cce64fed6eabb56af3d145cb45 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:59:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,665 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in 1fc1c9cce64fed6eabb56af3d145cb45 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:59:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:52,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/7b60682c706c471c930932b27366a2d2 is 50, key is test_row_0/C:col10/1733201992164/Put/seqid=0 2024-12-03T04:59:52,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741896_1072 (size=12301) 2024-12-03T04:59:52,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:59:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:52,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:52,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:59:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202052820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202052821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:52,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:59:52,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:52,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:52,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:52,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:52,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:53,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:59:53,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:53,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:53,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:53,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:53,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:59:53,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/7b60682c706c471c930932b27366a2d2 2024-12-03T04:59:53,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/d9e086aa8c18441a9843bcd091828f8a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d9e086aa8c18441a9843bcd091828f8a 2024-12-03T04:59:53,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d9e086aa8c18441a9843bcd091828f8a, entries=200, sequenceid=296, filesize=14.4 K 2024-12-03T04:59:53,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/84eccff191aa4cfe82f8ea2c1bdc0301 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/84eccff191aa4cfe82f8ea2c1bdc0301 2024-12-03T04:59:53,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/84eccff191aa4cfe82f8ea2c1bdc0301, entries=150, sequenceid=296, filesize=12.0 K 2024-12-03T04:59:53,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/7b60682c706c471c930932b27366a2d2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/7b60682c706c471c930932b27366a2d2 2024-12-03T04:59:53,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/7b60682c706c471c930932b27366a2d2, entries=150, sequenceid=296, filesize=12.0 K 2024-12-03T04:59:53,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 1fc1c9cce64fed6eabb56af3d145cb45 in 974ms, sequenceid=296, compaction requested=true 2024-12-03T04:59:53,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:53,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:59:53,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:53,140 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:59:53,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:59:53,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:53,140 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:59:53,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:59:53,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:53,144 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56670 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:59:53,144 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/A is initiating minor compaction (all files) 2024-12-03T04:59:53,145 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/A in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:53,145 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/05a0930c9a00455199dfc6f9df82d6a6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7c5dba6f424048ce9419f665246fe165, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6447ca6c63c24d969f061661ec5cabbc, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d9e086aa8c18441a9843bcd091828f8a] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=55.3 K 2024-12-03T04:59:53,148 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:59:53,148 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/B is initiating minor compaction (all files) 2024-12-03T04:59:53,148 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05a0930c9a00455199dfc6f9df82d6a6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201989703 2024-12-03T04:59:53,148 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/B in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:53,148 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/9b39e91e24f74b51b4ebba013a464059, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/05cac77ed7264ebeacb9dcc0b2ab4094, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5bf923e076fe47b2b4066f93c5df9514, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/84eccff191aa4cfe82f8ea2c1bdc0301] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=48.3 K 2024-12-03T04:59:53,150 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b39e91e24f74b51b4ebba013a464059, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201989703 2024-12-03T04:59:53,150 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c5dba6f424048ce9419f665246fe165, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733201990427 2024-12-03T04:59:53,151 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 05cac77ed7264ebeacb9dcc0b2ab4094, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733201990427 2024-12-03T04:59:53,151 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6447ca6c63c24d969f061661ec5cabbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733201991104 2024-12-03T04:59:53,152 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bf923e076fe47b2b4066f93c5df9514, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733201991104 2024-12-03T04:59:53,152 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9e086aa8c18441a9843bcd091828f8a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201991514 2024-12-03T04:59:53,153 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 84eccff191aa4cfe82f8ea2c1bdc0301, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201991514 2024-12-03T04:59:53,176 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:53,177 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/a0aa15cbadab42e68e69c44b74752687 is 50, key is test_row_0/A:col10/1733201992164/Put/seqid=0 2024-12-03T04:59:53,188 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:53,189 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/d9f4fcb7a24d4c74a1c1946f877e38c2 is 50, key is test_row_0/B:col10/1733201992164/Put/seqid=0 2024-12-03T04:59:53,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:59:53,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:53,214 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-03T04:59:53,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741897_1073 (size=12983) 2024-12-03T04:59:53,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/13deb3b63b554b0cb7a244f8edc8e688 is 50, key is test_row_0/A:col10/1733201992181/Put/seqid=0 2024-12-03T04:59:53,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741899_1075 (size=12301) 2024-12-03T04:59:53,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741898_1074 (size=12983) 2024-12-03T04:59:53,244 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/d9f4fcb7a24d4c74a1c1946f877e38c2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d9f4fcb7a24d4c74a1c1946f877e38c2 2024-12-03T04:59:53,255 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/B of 1fc1c9cce64fed6eabb56af3d145cb45 into d9f4fcb7a24d4c74a1c1946f877e38c2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:53,255 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:53,255 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/B, priority=12, startTime=1733201993140; duration=0sec 2024-12-03T04:59:53,255 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:53,255 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:B 2024-12-03T04:59:53,256 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:59:53,266 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:59:53,266 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/C is initiating minor compaction (all files) 2024-12-03T04:59:53,266 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/C in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:53,267 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/94d09ec6438241e787c192731266afd5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1647dc26dfa14007ba1dfd823823fd36, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/6899fdf3eb93487c8804e14751968515, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/7b60682c706c471c930932b27366a2d2] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=48.3 K 2024-12-03T04:59:53,267 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d09ec6438241e787c192731266afd5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201989703 2024-12-03T04:59:53,267 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1647dc26dfa14007ba1dfd823823fd36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733201990427 2024-12-03T04:59:53,268 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6899fdf3eb93487c8804e14751968515, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733201991104 2024-12-03T04:59:53,270 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b60682c706c471c930932b27366a2d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201991514 2024-12-03T04:59:53,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:53,288 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#C#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:53,289 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/1cd06ed39268416fbe70751af0f564c9 is 50, key is test_row_0/C:col10/1733201992164/Put/seqid=0 2024-12-03T04:59:53,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741900_1076 (size=12983) 2024-12-03T04:59:53,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202053385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:53,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202053389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:53,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202053394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:53,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:53,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202053501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:53,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202053510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:53,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202053512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:59:53,634 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/13deb3b63b554b0cb7a244f8edc8e688 2024-12-03T04:59:53,636 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/a0aa15cbadab42e68e69c44b74752687 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/a0aa15cbadab42e68e69c44b74752687 2024-12-03T04:59:53,645 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/A of 1fc1c9cce64fed6eabb56af3d145cb45 into a0aa15cbadab42e68e69c44b74752687(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:53,645 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:53,645 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/A, priority=12, startTime=1733201993139; duration=0sec 2024-12-03T04:59:53,645 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:53,646 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:A 2024-12-03T04:59:53,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/ddfe2e577cd04f8cab06c5ae67e75027 is 50, key is test_row_0/B:col10/1733201992181/Put/seqid=0 2024-12-03T04:59:53,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741901_1077 (size=12301) 2024-12-03T04:59:53,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202053711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:53,725 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/1cd06ed39268416fbe70751af0f564c9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1cd06ed39268416fbe70751af0f564c9 2024-12-03T04:59:53,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202053721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:53,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202053721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:53,738 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/C of 1fc1c9cce64fed6eabb56af3d145cb45 into 1cd06ed39268416fbe70751af0f564c9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:53,738 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:53,738 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/C, priority=12, startTime=1733201993140; duration=0sec 2024-12-03T04:59:53,738 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:53,738 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:C 2024-12-03T04:59:54,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202054020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:54,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:54,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202054040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202054041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:54,079 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/ddfe2e577cd04f8cab06c5ae67e75027 2024-12-03T04:59:54,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/2f46739e5dfa4c33a5ad94e175c27ee7 is 50, key is test_row_0/C:col10/1733201992181/Put/seqid=0 2024-12-03T04:59:54,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741902_1078 (size=12301) 2024-12-03T04:59:54,505 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/2f46739e5dfa4c33a5ad94e175c27ee7 2024-12-03T04:59:54,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/13deb3b63b554b0cb7a244f8edc8e688 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/13deb3b63b554b0cb7a244f8edc8e688 2024-12-03T04:59:54,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202054531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:54,547 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/13deb3b63b554b0cb7a244f8edc8e688, entries=150, sequenceid=305, filesize=12.0 K 2024-12-03T04:59:54,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/ddfe2e577cd04f8cab06c5ae67e75027 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/ddfe2e577cd04f8cab06c5ae67e75027 2024-12-03T04:59:54,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202054551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:54,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202054556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:54,578 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/ddfe2e577cd04f8cab06c5ae67e75027, entries=150, sequenceid=305, filesize=12.0 K 2024-12-03T04:59:54,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/2f46739e5dfa4c33a5ad94e175c27ee7 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2f46739e5dfa4c33a5ad94e175c27ee7 2024-12-03T04:59:54,586 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2f46739e5dfa4c33a5ad94e175c27ee7, entries=150, sequenceid=305, filesize=12.0 K 2024-12-03T04:59:54,588 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 1fc1c9cce64fed6eabb56af3d145cb45 in 1375ms, sequenceid=305, compaction requested=false 2024-12-03T04:59:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-03T04:59:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-03T04:59:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:59:54,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T04:59:54,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1510 sec 2024-12-03T04:59:54,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.1650 sec 2024-12-03T04:59:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:54,694 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:54,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-12-03T04:59:54,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/75a1dc5687d34c02bb501e7de4c35e1e is 50, key is test_row_0/A:col10/1733201993387/Put/seqid=0 2024-12-03T04:59:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202054709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:54,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202054713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:54,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741903_1079 (size=14741) 2024-12-03T04:59:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202054821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:54,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202054821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202055031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202055031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/75a1dc5687d34c02bb501e7de4c35e1e 2024-12-03T04:59:55,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/acc4c91ce97e4fd3a16c008a40540e39 is 50, key is test_row_0/B:col10/1733201993387/Put/seqid=0 2024-12-03T04:59:55,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741904_1080 (size=12301) 2024-12-03T04:59:55,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/acc4c91ce97e4fd3a16c008a40540e39 2024-12-03T04:59:55,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/31af8f78c4284241a985339fab9a63ca is 50, key is test_row_0/C:col10/1733201993387/Put/seqid=0 2024-12-03T04:59:55,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741905_1081 (size=12301) 2024-12-03T04:59:55,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/31af8f78c4284241a985339fab9a63ca 2024-12-03T04:59:55,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/75a1dc5687d34c02bb501e7de4c35e1e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/75a1dc5687d34c02bb501e7de4c35e1e 2024-12-03T04:59:55,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/75a1dc5687d34c02bb501e7de4c35e1e, entries=200, sequenceid=336, filesize=14.4 K 2024-12-03T04:59:55,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/acc4c91ce97e4fd3a16c008a40540e39 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/acc4c91ce97e4fd3a16c008a40540e39 2024-12-03T04:59:55,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/acc4c91ce97e4fd3a16c008a40540e39, entries=150, sequenceid=336, filesize=12.0 K 2024-12-03T04:59:55,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/31af8f78c4284241a985339fab9a63ca as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/31af8f78c4284241a985339fab9a63ca 2024-12-03T04:59:55,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/31af8f78c4284241a985339fab9a63ca, entries=150, sequenceid=336, filesize=12.0 K 2024-12-03T04:59:55,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 1fc1c9cce64fed6eabb56af3d145cb45 in 563ms, sequenceid=336, compaction requested=true 2024-12-03T04:59:55,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:55,257 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:55,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:59:55,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:55,258 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:55,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:59:55,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:55,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:59:55,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:55,262 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:55,263 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/A is initiating minor compaction (all files) 2024-12-03T04:59:55,263 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/A in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:55,263 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/a0aa15cbadab42e68e69c44b74752687, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/13deb3b63b554b0cb7a244f8edc8e688, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/75a1dc5687d34c02bb501e7de4c35e1e] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=39.1 K 2024-12-03T04:59:55,264 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:55,264 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0aa15cbadab42e68e69c44b74752687, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201991514 2024-12-03T04:59:55,264 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/B is initiating minor compaction (all files) 2024-12-03T04:59:55,264 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/B in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:55,265 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d9f4fcb7a24d4c74a1c1946f877e38c2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/ddfe2e577cd04f8cab06c5ae67e75027, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/acc4c91ce97e4fd3a16c008a40540e39] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=36.7 K 2024-12-03T04:59:55,265 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13deb3b63b554b0cb7a244f8edc8e688, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733201992176 2024-12-03T04:59:55,265 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting d9f4fcb7a24d4c74a1c1946f877e38c2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201991514 2024-12-03T04:59:55,266 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75a1dc5687d34c02bb501e7de4c35e1e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733201993380 2024-12-03T04:59:55,266 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ddfe2e577cd04f8cab06c5ae67e75027, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733201992176 2024-12-03T04:59:55,266 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting acc4c91ce97e4fd3a16c008a40540e39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733201993384 2024-12-03T04:59:55,292 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:55,292 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:55,293 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/d5cb3310055b46618cbfd328aa4be79b is 50, key is test_row_0/B:col10/1733201993387/Put/seqid=0 2024-12-03T04:59:55,293 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/15b182a270f44eb5b6f19e076d9af67c is 50, key is test_row_0/A:col10/1733201993387/Put/seqid=0 2024-12-03T04:59:55,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741906_1082 (size=13085) 2024-12-03T04:59:55,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741907_1083 (size=13085) 2024-12-03T04:59:55,320 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/15b182a270f44eb5b6f19e076d9af67c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/15b182a270f44eb5b6f19e076d9af67c 2024-12-03T04:59:55,327 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/A of 1fc1c9cce64fed6eabb56af3d145cb45 into 15b182a270f44eb5b6f19e076d9af67c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:55,328 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:55,328 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/A, priority=13, startTime=1733201995257; duration=0sec 2024-12-03T04:59:55,328 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:55,328 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:A 2024-12-03T04:59:55,328 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:55,331 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:55,331 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/C is initiating minor compaction (all files) 2024-12-03T04:59:55,331 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/C in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:55,331 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1cd06ed39268416fbe70751af0f564c9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2f46739e5dfa4c33a5ad94e175c27ee7, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/31af8f78c4284241a985339fab9a63ca] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=36.7 K 2024-12-03T04:59:55,332 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cd06ed39268416fbe70751af0f564c9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201991514 2024-12-03T04:59:55,335 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f46739e5dfa4c33a5ad94e175c27ee7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733201992176 2024-12-03T04:59:55,335 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31af8f78c4284241a985339fab9a63ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733201993384 2024-12-03T04:59:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:55,347 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:55,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T04:59:55,350 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:55,352 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/fbb792de09bd43fdacaf0b2b003de00f is 50, key is test_row_0/C:col10/1733201993387/Put/seqid=0 2024-12-03T04:59:55,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/40efe9a84e154c29a9f50525bb552b4b is 50, key is test_row_0/A:col10/1733201995345/Put/seqid=0 2024-12-03T04:59:55,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741909_1085 (size=14741) 2024-12-03T04:59:55,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741908_1084 (size=13085) 2024-12-03T04:59:55,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202055420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202055424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202055530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202055531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202055551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:55,555 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_1' in 1fc1c9cce64fed6eabb56af3d145cb45 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:59:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202055572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:55,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202055573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:55,576 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_1' in 1fc1c9cce64fed6eabb56af3d145cb45 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:59:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,577 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in 1fc1c9cce64fed6eabb56af3d145cb45 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:59:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,716 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/d5cb3310055b46618cbfd328aa4be79b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d5cb3310055b46618cbfd328aa4be79b 2024-12-03T04:59:55,727 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/B of 1fc1c9cce64fed6eabb56af3d145cb45 into d5cb3310055b46618cbfd328aa4be79b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:55,729 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:55,729 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/B, priority=13, startTime=1733201995258; duration=0sec 2024-12-03T04:59:55,729 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:55,729 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:B 2024-12-03T04:59:55,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202055741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202055741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:55,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/40efe9a84e154c29a9f50525bb552b4b 2024-12-03T04:59:55,804 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/fbb792de09bd43fdacaf0b2b003de00f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/fbb792de09bd43fdacaf0b2b003de00f 2024-12-03T04:59:55,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/c64c969879864d99bbfc1983bf513d87 is 50, key is test_row_0/B:col10/1733201995345/Put/seqid=0 2024-12-03T04:59:55,816 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/C of 1fc1c9cce64fed6eabb56af3d145cb45 into fbb792de09bd43fdacaf0b2b003de00f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:55,816 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:55,816 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/C, priority=13, startTime=1733201995258; duration=0sec 2024-12-03T04:59:55,816 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:55,817 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:C 2024-12-03T04:59:55,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741910_1086 (size=12301) 2024-12-03T04:59:56,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202056061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202056061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:56,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/c64c969879864d99bbfc1983bf513d87 2024-12-03T04:59:56,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/b394b2286ddc41a3b5dd76cbe17d9af9 is 50, key is test_row_0/C:col10/1733201995345/Put/seqid=0 2024-12-03T04:59:56,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741911_1087 (size=12301) 2024-12-03T04:59:56,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/b394b2286ddc41a3b5dd76cbe17d9af9 2024-12-03T04:59:56,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/40efe9a84e154c29a9f50525bb552b4b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/40efe9a84e154c29a9f50525bb552b4b 2024-12-03T04:59:56,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/40efe9a84e154c29a9f50525bb552b4b, entries=200, sequenceid=348, filesize=14.4 K 2024-12-03T04:59:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/c64c969879864d99bbfc1983bf513d87 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c64c969879864d99bbfc1983bf513d87 2024-12-03T04:59:56,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c64c969879864d99bbfc1983bf513d87, entries=150, sequenceid=348, filesize=12.0 K 2024-12-03T04:59:56,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/b394b2286ddc41a3b5dd76cbe17d9af9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/b394b2286ddc41a3b5dd76cbe17d9af9 2024-12-03T04:59:56,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/b394b2286ddc41a3b5dd76cbe17d9af9, entries=150, sequenceid=348, filesize=12.0 K 2024-12-03T04:59:56,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1fc1c9cce64fed6eabb56af3d145cb45 in 999ms, sequenceid=348, compaction requested=false 2024-12-03T04:59:56,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:56,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:56,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-03T04:59:56,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/ec4e43ce96fb441080e36d5656b2ec71 is 50, key is test_row_0/A:col10/1733201996576/Put/seqid=0 2024-12-03T04:59:56,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741912_1088 (size=14741) 2024-12-03T04:59:56,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/ec4e43ce96fb441080e36d5656b2ec71 2024-12-03T04:59:56,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202056593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:56,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202056597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:56,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/962ce83e90284ecd97204b1769ef9854 is 50, key is test_row_0/B:col10/1733201996576/Put/seqid=0 2024-12-03T04:59:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:59:56,611 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:59:56,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:59:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-03T04:59:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:59:56,615 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:59:56,616 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:59:56,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:59:56,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741913_1089 (size=12301) 2024-12-03T04:59:56,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/962ce83e90284ecd97204b1769ef9854 2024-12-03T04:59:56,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/339572eca14b422b80ea125ed1bba32c is 50, key is test_row_0/C:col10/1733201996576/Put/seqid=0 2024-12-03T04:59:56,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741914_1090 (size=12301) 2024-12-03T04:59:56,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202056700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:56,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202056710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:59:56,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:59:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:56,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:56,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202056911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:56,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202056920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:56,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:59:56,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:56,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:56,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:56,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:56,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:59:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/339572eca14b422b80ea125ed1bba32c 2024-12-03T04:59:57,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/ec4e43ce96fb441080e36d5656b2ec71 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ec4e43ce96fb441080e36d5656b2ec71 2024-12-03T04:59:57,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ec4e43ce96fb441080e36d5656b2ec71, entries=200, sequenceid=377, filesize=14.4 K 2024-12-03T04:59:57,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/962ce83e90284ecd97204b1769ef9854 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/962ce83e90284ecd97204b1769ef9854 2024-12-03T04:59:57,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:59:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:57,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/962ce83e90284ecd97204b1769ef9854, entries=150, sequenceid=377, filesize=12.0 K 2024-12-03T04:59:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/339572eca14b422b80ea125ed1bba32c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/339572eca14b422b80ea125ed1bba32c 2024-12-03T04:59:57,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/339572eca14b422b80ea125ed1bba32c, entries=150, sequenceid=377, filesize=12.0 K 2024-12-03T04:59:57,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 1fc1c9cce64fed6eabb56af3d145cb45 in 523ms, sequenceid=377, compaction requested=true 2024-12-03T04:59:57,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:57,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:59:57,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:57,101 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:57,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:59:57,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:57,101 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:57,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:59:57,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:57,103 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:57,104 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/A is initiating minor compaction (all files) 2024-12-03T04:59:57,104 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/A in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,104 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/15b182a270f44eb5b6f19e076d9af67c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/40efe9a84e154c29a9f50525bb552b4b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ec4e43ce96fb441080e36d5656b2ec71] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=41.6 K 2024-12-03T04:59:57,104 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15b182a270f44eb5b6f19e076d9af67c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733201993384 2024-12-03T04:59:57,104 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:57,105 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/B is initiating minor compaction (all files) 2024-12-03T04:59:57,105 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/B in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,105 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d5cb3310055b46618cbfd328aa4be79b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c64c969879864d99bbfc1983bf513d87, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/962ce83e90284ecd97204b1769ef9854] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=36.8 K 2024-12-03T04:59:57,106 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40efe9a84e154c29a9f50525bb552b4b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733201994711 2024-12-03T04:59:57,106 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec4e43ce96fb441080e36d5656b2ec71, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733201995408 2024-12-03T04:59:57,106 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting d5cb3310055b46618cbfd328aa4be79b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733201993384 2024-12-03T04:59:57,113 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c64c969879864d99bbfc1983bf513d87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733201994711 2024-12-03T04:59:57,114 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 962ce83e90284ecd97204b1769ef9854, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733201995408 2024-12-03T04:59:57,129 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:57,129 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/c466f8772fcc45c9b59d10e736720fbf is 50, key is test_row_0/A:col10/1733201996576/Put/seqid=0 2024-12-03T04:59:57,130 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:57,130 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/8386188eac5d41b2a02b78ecb89fce3a is 50, key is test_row_0/B:col10/1733201996576/Put/seqid=0 2024-12-03T04:59:57,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741915_1091 (size=13187) 2024-12-03T04:59:57,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741916_1092 (size=13187) 2024-12-03T04:59:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:57,238 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:57,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T04:59:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:59:57,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:59:57,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:57,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/7f4139cbcea842819b6bbb040e3c9c2c is 50, key is test_row_0/A:col10/1733201997222/Put/seqid=0 2024-12-03T04:59:57,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741917_1093 (size=12301) 2024-12-03T04:59:57,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202057333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:57,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202057337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:57,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:59:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202057451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:57,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202057451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:57,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:59:57,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43002 deadline: 1733202057571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:57,573 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_1' in 1fc1c9cce64fed6eabb56af3d145cb45 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:59:57,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:57,586 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/c466f8772fcc45c9b59d10e736720fbf as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/c466f8772fcc45c9b59d10e736720fbf 2024-12-03T04:59:57,589 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/8386188eac5d41b2a02b78ecb89fce3a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/8386188eac5d41b2a02b78ecb89fce3a 2024-12-03T04:59:57,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42972 deadline: 1733202057591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:57,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:57,598 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in 1fc1c9cce64fed6eabb56af3d145cb45 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:59:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42956 deadline: 1733202057590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:57,599 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_1' in 1fc1c9cce64fed6eabb56af3d145cb45 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:59:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:57,606 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/A of 1fc1c9cce64fed6eabb56af3d145cb45 into c466f8772fcc45c9b59d10e736720fbf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:57,606 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:57,606 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/A, priority=13, startTime=1733201997101; duration=0sec 2024-12-03T04:59:57,607 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:57,607 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:A 2024-12-03T04:59:57,607 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:59:57,608 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:59:57,608 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/C is initiating minor compaction (all files) 2024-12-03T04:59:57,609 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/C in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,609 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/B of 1fc1c9cce64fed6eabb56af3d145cb45 into 8386188eac5d41b2a02b78ecb89fce3a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:57,609 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/fbb792de09bd43fdacaf0b2b003de00f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/b394b2286ddc41a3b5dd76cbe17d9af9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/339572eca14b422b80ea125ed1bba32c] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=36.8 K 2024-12-03T04:59:57,609 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:57,609 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/B, priority=13, startTime=1733201997101; duration=0sec 2024-12-03T04:59:57,609 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:57,609 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:B 2024-12-03T04:59:57,609 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb792de09bd43fdacaf0b2b003de00f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733201993384 2024-12-03T04:59:57,610 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b394b2286ddc41a3b5dd76cbe17d9af9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733201994711 2024-12-03T04:59:57,610 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 339572eca14b422b80ea125ed1bba32c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733201995408 2024-12-03T04:59:57,639 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#C#compaction#81 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:57,640 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/f21cb5a29fe54834b35b9fb4015dbe2a is 50, key is test_row_0/C:col10/1733201996576/Put/seqid=0 2024-12-03T04:59:57,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202057661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202057661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:57,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:57,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741918_1094 (size=13187) 2024-12-03T04:59:57,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/7f4139cbcea842819b6bbb040e3c9c2c 2024-12-03T04:59:57,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:59:57,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:57,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/5fb3d9a9d0814e11aa42047ff048e89f is 50, key is test_row_0/B:col10/1733201997222/Put/seqid=0 2024-12-03T04:59:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:59:57,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741919_1095 (size=12301) 2024-12-03T04:59:57,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/5fb3d9a9d0814e11aa42047ff048e89f 2024-12-03T04:59:57,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/fc04e46c3e3a4722be1dc43ebb61f8c8 is 50, key is test_row_0/C:col10/1733201997222/Put/seqid=0 2024-12-03T04:59:57,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741920_1096 (size=12301) 2024-12-03T04:59:57,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:59:57,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:57,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:57,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202057973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:57,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202057981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:58,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:59:58,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:58,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:58,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:58,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:58,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:58,092 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/f21cb5a29fe54834b35b9fb4015dbe2a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f21cb5a29fe54834b35b9fb4015dbe2a 2024-12-03T04:59:58,110 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/C of 1fc1c9cce64fed6eabb56af3d145cb45 into f21cb5a29fe54834b35b9fb4015dbe2a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:58,110 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:58,110 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/C, priority=13, startTime=1733201997101; duration=0sec 2024-12-03T04:59:58,111 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:58,111 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:C 2024-12-03T04:59:58,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/fc04e46c3e3a4722be1dc43ebb61f8c8 2024-12-03T04:59:58,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:59:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:58,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:59:58,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/7f4139cbcea842819b6bbb040e3c9c2c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7f4139cbcea842819b6bbb040e3c9c2c 2024-12-03T04:59:58,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7f4139cbcea842819b6bbb040e3c9c2c, entries=150, sequenceid=388, filesize=12.0 K 2024-12-03T04:59:58,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/5fb3d9a9d0814e11aa42047ff048e89f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5fb3d9a9d0814e11aa42047ff048e89f 2024-12-03T04:59:58,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5fb3d9a9d0814e11aa42047ff048e89f, entries=150, sequenceid=388, filesize=12.0 K 2024-12-03T04:59:58,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/fc04e46c3e3a4722be1dc43ebb61f8c8 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/fc04e46c3e3a4722be1dc43ebb61f8c8 2024-12-03T04:59:58,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/fc04e46c3e3a4722be1dc43ebb61f8c8, entries=150, sequenceid=388, filesize=12.0 K 2024-12-03T04:59:58,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1fc1c9cce64fed6eabb56af3d145cb45 in 1053ms, sequenceid=388, compaction requested=false 2024-12-03T04:59:58,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:58,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:59:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:58,391 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T04:59:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/71bb8f4354284e0caa9d395da898c3c1 is 50, key is test_row_0/A:col10/1733201997311/Put/seqid=0 2024-12-03T04:59:58,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741921_1097 (size=12301) 2024-12-03T04:59:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:58,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T04:59:58,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202058520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:58,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:58,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202058527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:58,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202058640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202058640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:59:58,850 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/71bb8f4354284e0caa9d395da898c3c1 2024-12-03T04:59:58,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:58,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202058850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202058851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:58,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/aa63e8ed9ac94e10bfb58e12fc882831 is 50, key is test_row_0/B:col10/1733201997311/Put/seqid=0 2024-12-03T04:59:58,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741922_1098 (size=12301) 2024-12-03T04:59:58,869 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/aa63e8ed9ac94e10bfb58e12fc882831 2024-12-03T04:59:58,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/f87339b0955e4e0ea67d8804d158a046 is 50, key is test_row_0/C:col10/1733201997311/Put/seqid=0 2024-12-03T04:59:58,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741923_1099 (size=12301) 2024-12-03T04:59:58,890 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/f87339b0955e4e0ea67d8804d158a046 2024-12-03T04:59:58,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/71bb8f4354284e0caa9d395da898c3c1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/71bb8f4354284e0caa9d395da898c3c1 2024-12-03T04:59:58,905 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/71bb8f4354284e0caa9d395da898c3c1, entries=150, sequenceid=416, filesize=12.0 K 2024-12-03T04:59:58,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/aa63e8ed9ac94e10bfb58e12fc882831 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/aa63e8ed9ac94e10bfb58e12fc882831 2024-12-03T04:59:58,916 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/aa63e8ed9ac94e10bfb58e12fc882831, entries=150, sequenceid=416, filesize=12.0 K 2024-12-03T04:59:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/f87339b0955e4e0ea67d8804d158a046 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f87339b0955e4e0ea67d8804d158a046 2024-12-03T04:59:58,926 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f87339b0955e4e0ea67d8804d158a046, entries=150, sequenceid=416, filesize=12.0 K 2024-12-03T04:59:58,934 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1fc1c9cce64fed6eabb56af3d145cb45 in 537ms, sequenceid=416, compaction requested=true 2024-12-03T04:59:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-03T04:59:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-03T04:59:58,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T04:59:58,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3230 sec 2024-12-03T04:59:58,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.3310 sec 2024-12-03T04:59:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:59,164 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:59,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-03T04:59:59,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/1898fbcced10470788a2ec1620b4ab1c is 50, key is test_row_0/A:col10/1733201998517/Put/seqid=0 2024-12-03T04:59:59,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741924_1100 (size=17181) 2024-12-03T04:59:59,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:59,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202059232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202059233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:59,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202059341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:59,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:59,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202059341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:59,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202059551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:59,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:59,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202059551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:59,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:59,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/1898fbcced10470788a2ec1620b4ab1c 2024-12-03T04:59:59,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/09b36222df784782a0334aacd2da1d7f is 50, key is test_row_0/B:col10/1733201998517/Put/seqid=0 2024-12-03T04:59:59,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741925_1101 (size=12301) 2024-12-03T04:59:59,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/09b36222df784782a0334aacd2da1d7f 2024-12-03T04:59:59,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/1ae0b7955eeb463bbddb806c4c98beb0 is 50, key is test_row_0/C:col10/1733201998517/Put/seqid=0 2024-12-03T04:59:59,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741926_1102 (size=12301) 2024-12-03T04:59:59,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/1ae0b7955eeb463bbddb806c4c98beb0 2024-12-03T04:59:59,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/1898fbcced10470788a2ec1620b4ab1c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/1898fbcced10470788a2ec1620b4ab1c 2024-12-03T04:59:59,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/1898fbcced10470788a2ec1620b4ab1c, entries=250, sequenceid=429, filesize=16.8 K 2024-12-03T04:59:59,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/09b36222df784782a0334aacd2da1d7f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/09b36222df784782a0334aacd2da1d7f 2024-12-03T04:59:59,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/09b36222df784782a0334aacd2da1d7f, entries=150, sequenceid=429, filesize=12.0 K 2024-12-03T04:59:59,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/1ae0b7955eeb463bbddb806c4c98beb0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1ae0b7955eeb463bbddb806c4c98beb0 2024-12-03T04:59:59,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1ae0b7955eeb463bbddb806c4c98beb0, entries=150, sequenceid=429, filesize=12.0 K 2024-12-03T04:59:59,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1fc1c9cce64fed6eabb56af3d145cb45 in 586ms, sequenceid=429, compaction requested=true 2024-12-03T04:59:59,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:59,750 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:59:59,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:59:59,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:59,752 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:59:59,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:59:59,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:59,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:59:59,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:59,752 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:59:59,752 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/A is initiating minor compaction (all files) 2024-12-03T04:59:59,753 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/A in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:59,753 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/c466f8772fcc45c9b59d10e736720fbf, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7f4139cbcea842819b6bbb040e3c9c2c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/71bb8f4354284e0caa9d395da898c3c1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/1898fbcced10470788a2ec1620b4ab1c] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=53.7 K 2024-12-03T04:59:59,753 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c466f8772fcc45c9b59d10e736720fbf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733201995408 2024-12-03T04:59:59,755 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f4139cbcea842819b6bbb040e3c9c2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733201996589 2024-12-03T04:59:59,755 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:59:59,755 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/B is initiating minor compaction (all files) 2024-12-03T04:59:59,755 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71bb8f4354284e0caa9d395da898c3c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733201997311 2024-12-03T04:59:59,756 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/B in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:59,756 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/8386188eac5d41b2a02b78ecb89fce3a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5fb3d9a9d0814e11aa42047ff048e89f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/aa63e8ed9ac94e10bfb58e12fc882831, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/09b36222df784782a0334aacd2da1d7f] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=48.9 K 2024-12-03T04:59:59,756 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1898fbcced10470788a2ec1620b4ab1c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733201998511 2024-12-03T04:59:59,757 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 8386188eac5d41b2a02b78ecb89fce3a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733201995408 2024-12-03T04:59:59,761 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fb3d9a9d0814e11aa42047ff048e89f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733201996589 2024-12-03T04:59:59,762 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting aa63e8ed9ac94e10bfb58e12fc882831, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733201997311 2024-12-03T04:59:59,763 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 09b36222df784782a0334aacd2da1d7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733201998517 2024-12-03T04:59:59,770 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:59,771 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/8614891331be45929cf4b7c5f05a4634 is 50, key is test_row_0/A:col10/1733201998517/Put/seqid=0 2024-12-03T04:59:59,790 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:59,791 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/109575e3153e4bada1e41a9cd6cbad45 is 50, key is test_row_0/B:col10/1733201998517/Put/seqid=0 2024-12-03T04:59:59,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741928_1104 (size=13323) 2024-12-03T04:59:59,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741927_1103 (size=13323) 2024-12-03T04:59:59,812 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/8614891331be45929cf4b7c5f05a4634 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8614891331be45929cf4b7c5f05a4634 2024-12-03T04:59:59,813 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/109575e3153e4bada1e41a9cd6cbad45 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/109575e3153e4bada1e41a9cd6cbad45 2024-12-03T04:59:59,821 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/A of 1fc1c9cce64fed6eabb56af3d145cb45 into 8614891331be45929cf4b7c5f05a4634(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:59,821 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:59,821 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/A, priority=12, startTime=1733201999750; duration=0sec 2024-12-03T04:59:59,821 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:59:59,822 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:A 2024-12-03T04:59:59,822 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:59:59,823 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:59:59,823 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/C is initiating minor compaction (all files) 2024-12-03T04:59:59,824 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/C in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T04:59:59,824 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f21cb5a29fe54834b35b9fb4015dbe2a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/fc04e46c3e3a4722be1dc43ebb61f8c8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f87339b0955e4e0ea67d8804d158a046, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1ae0b7955eeb463bbddb806c4c98beb0] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=48.9 K 2024-12-03T04:59:59,824 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f21cb5a29fe54834b35b9fb4015dbe2a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733201995408 2024-12-03T04:59:59,825 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc04e46c3e3a4722be1dc43ebb61f8c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733201996589 2024-12-03T04:59:59,825 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f87339b0955e4e0ea67d8804d158a046, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733201997311 2024-12-03T04:59:59,826 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/B of 1fc1c9cce64fed6eabb56af3d145cb45 into 109575e3153e4bada1e41a9cd6cbad45(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:59:59,826 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T04:59:59,826 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/B, priority=12, startTime=1733201999751; duration=0sec 2024-12-03T04:59:59,826 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae0b7955eeb463bbddb806c4c98beb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733201998517 2024-12-03T04:59:59,826 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:59:59,826 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:B 2024-12-03T04:59:59,844 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#C#compaction#92 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:59:59,845 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/868a78befab6473cb2018664d96f1611 is 50, key is test_row_0/C:col10/1733201998517/Put/seqid=0 2024-12-03T04:59:59,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741929_1105 (size=13323) 2024-12-03T04:59:59,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T04:59:59,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-03T04:59:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T04:59:59,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/f695307faf20433790a3d8a3c9920688 is 50, key is test_row_0/A:col10/1733201999222/Put/seqid=0 2024-12-03T04:59:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202059890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:59,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:59:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202059900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T04:59:59,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:59:59,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:59:59,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741930_1106 (size=12301) 2024-12-03T04:59:59,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/f695307faf20433790a3d8a3c9920688 2024-12-03T04:59:59,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/33a92d9a6e6f400cb1f76407943655a3 is 50, key is test_row_0/B:col10/1733201999222/Put/seqid=0 2024-12-03T04:59:59,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741931_1107 (size=12301) 2024-12-03T04:59:59,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/33a92d9a6e6f400cb1f76407943655a3 2024-12-03T05:00:00,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/80b36128eb324bd5adbb6e67395d269c is 50, key is test_row_0/C:col10/1733201999222/Put/seqid=0 2024-12-03T05:00:00,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202060011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:00,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202060011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:00,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741932_1108 (size=12301) 2024-12-03T05:00:00,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202060221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:00,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202060221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:00,273 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/868a78befab6473cb2018664d96f1611 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/868a78befab6473cb2018664d96f1611 2024-12-03T05:00:00,288 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/C of 1fc1c9cce64fed6eabb56af3d145cb45 into 868a78befab6473cb2018664d96f1611(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:00,288 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T05:00:00,288 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/C, priority=12, startTime=1733201999752; duration=0sec 2024-12-03T05:00:00,288 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:00,288 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:C 2024-12-03T05:00:00,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/80b36128eb324bd5adbb6e67395d269c 2024-12-03T05:00:00,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/f695307faf20433790a3d8a3c9920688 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/f695307faf20433790a3d8a3c9920688 2024-12-03T05:00:00,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/f695307faf20433790a3d8a3c9920688, entries=150, sequenceid=455, filesize=12.0 K 2024-12-03T05:00:00,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/33a92d9a6e6f400cb1f76407943655a3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/33a92d9a6e6f400cb1f76407943655a3 2024-12-03T05:00:00,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/33a92d9a6e6f400cb1f76407943655a3, entries=150, sequenceid=455, filesize=12.0 K 2024-12-03T05:00:00,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/80b36128eb324bd5adbb6e67395d269c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/80b36128eb324bd5adbb6e67395d269c 2024-12-03T05:00:00,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/80b36128eb324bd5adbb6e67395d269c, entries=150, sequenceid=455, filesize=12.0 K 2024-12-03T05:00:00,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1fc1c9cce64fed6eabb56af3d145cb45 in 637ms, sequenceid=455, compaction requested=false 2024-12-03T05:00:00,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T05:00:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T05:00:00,536 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:00,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-03T05:00:00,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/d103dc9ab9674820acf582a6ab9b3475 is 50, key is test_row_0/A:col10/1733202000536/Put/seqid=0 2024-12-03T05:00:00,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741933_1109 (size=17181) 2024-12-03T05:00:00,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202060590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:00,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202060596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:00,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202060700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:00,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:00,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202060700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:00,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T05:00:00,774 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:00,775 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-03T05:00:00,777 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:00,779 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:00,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:00:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:00:00,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202060914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:00,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202060917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:00,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T05:00:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T05:00:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:00,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:00,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:00,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/d103dc9ab9674820acf582a6ab9b3475 2024-12-03T05:00:01,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/c5ea719c8e3e4a66bae20b3028915d6c is 50, key is test_row_0/B:col10/1733202000536/Put/seqid=0 2024-12-03T05:00:01,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741934_1110 (size=12301) 2024-12-03T05:00:01,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/c5ea719c8e3e4a66bae20b3028915d6c 2024-12-03T05:00:01,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/9945000a685b4e25a1ba90d6cd684094 is 50, key is test_row_0/C:col10/1733202000536/Put/seqid=0 2024-12-03T05:00:01,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741935_1111 (size=12301) 2024-12-03T05:00:01,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T05:00:01,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:01,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T05:00:01,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:01,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:01,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:00:01,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42954 deadline: 1733202061230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:01,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1733202061231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:01,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T05:00:01,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:01,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T05:00:01,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:01,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:01,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:01,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T05:00:01,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:01,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T05:00:01,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:01,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:01,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:00:01,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/9945000a685b4e25a1ba90d6cd684094 2024-12-03T05:00:01,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/d103dc9ab9674820acf582a6ab9b3475 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d103dc9ab9674820acf582a6ab9b3475 2024-12-03T05:00:01,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d103dc9ab9674820acf582a6ab9b3475, entries=250, sequenceid=471, filesize=16.8 K 2024-12-03T05:00:01,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/c5ea719c8e3e4a66bae20b3028915d6c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c5ea719c8e3e4a66bae20b3028915d6c 2024-12-03T05:00:01,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c5ea719c8e3e4a66bae20b3028915d6c, entries=150, sequenceid=471, filesize=12.0 K 2024-12-03T05:00:01,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/9945000a685b4e25a1ba90d6cd684094 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/9945000a685b4e25a1ba90d6cd684094 2024-12-03T05:00:01,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/9945000a685b4e25a1ba90d6cd684094, entries=150, sequenceid=471, filesize=12.0 K 2024-12-03T05:00:01,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1fc1c9cce64fed6eabb56af3d145cb45 in 947ms, sequenceid=471, compaction requested=true 2024-12-03T05:00:01,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T05:00:01,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:01,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:01,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:01,483 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:01,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:01,484 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:01,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fc1c9cce64fed6eabb56af3d145cb45:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:01,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:01,485 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:01,485 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:01,485 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/A is initiating minor compaction (all files) 2024-12-03T05:00:01,485 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/B is initiating minor compaction (all files) 2024-12-03T05:00:01,485 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/A in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:01,485 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/B in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:01,485 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8614891331be45929cf4b7c5f05a4634, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/f695307faf20433790a3d8a3c9920688, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d103dc9ab9674820acf582a6ab9b3475] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=41.8 K 2024-12-03T05:00:01,485 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/109575e3153e4bada1e41a9cd6cbad45, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/33a92d9a6e6f400cb1f76407943655a3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c5ea719c8e3e4a66bae20b3028915d6c] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=37.0 K 2024-12-03T05:00:01,486 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 109575e3153e4bada1e41a9cd6cbad45, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733201998517 2024-12-03T05:00:01,486 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8614891331be45929cf4b7c5f05a4634, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733201998517 2024-12-03T05:00:01,486 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 33a92d9a6e6f400cb1f76407943655a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1733201999214 2024-12-03T05:00:01,486 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f695307faf20433790a3d8a3c9920688, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1733201999214 2024-12-03T05:00:01,487 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ea719c8e3e4a66bae20b3028915d6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733201999897 2024-12-03T05:00:01,487 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d103dc9ab9674820acf582a6ab9b3475, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733201999887 2024-12-03T05:00:01,511 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:01,513 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/369e6f72d9824c27b82b06789de17c60 is 50, key is test_row_0/A:col10/1733202000536/Put/seqid=0 2024-12-03T05:00:01,515 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#B#compaction#100 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:01,516 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/1ce8eb9fa7de4c4cbf1c8298af332ecc is 50, key is test_row_0/B:col10/1733202000536/Put/seqid=0 2024-12-03T05:00:01,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741936_1112 (size=13425) 2024-12-03T05:00:01,526 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/369e6f72d9824c27b82b06789de17c60 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/369e6f72d9824c27b82b06789de17c60 2024-12-03T05:00:01,532 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/A of 1fc1c9cce64fed6eabb56af3d145cb45 into 369e6f72d9824c27b82b06789de17c60(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:01,532 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T05:00:01,532 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/A, priority=13, startTime=1733202001483; duration=0sec 2024-12-03T05:00:01,532 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:01,532 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:A 2024-12-03T05:00:01,533 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:01,535 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:01,535 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1fc1c9cce64fed6eabb56af3d145cb45/C is initiating minor compaction (all files) 2024-12-03T05:00:01,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741937_1113 (size=13425) 2024-12-03T05:00:01,538 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fc1c9cce64fed6eabb56af3d145cb45/C in TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:01,538 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/868a78befab6473cb2018664d96f1611, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/80b36128eb324bd5adbb6e67395d269c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/9945000a685b4e25a1ba90d6cd684094] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp, totalSize=37.0 K 2024-12-03T05:00:01,540 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 868a78befab6473cb2018664d96f1611, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733201998517 2024-12-03T05:00:01,541 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b36128eb324bd5adbb6e67395d269c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1733201999214 2024-12-03T05:00:01,541 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9945000a685b4e25a1ba90d6cd684094, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733201999897 2024-12-03T05:00:01,549 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/1ce8eb9fa7de4c4cbf1c8298af332ecc as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/1ce8eb9fa7de4c4cbf1c8298af332ecc 2024-12-03T05:00:01,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T05:00:01,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:01,551 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-03T05:00:01,556 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fc1c9cce64fed6eabb56af3d145cb45#C#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:01,557 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/f1eeaf9015934d74af25c4d4e2eb485d is 50, key is test_row_0/C:col10/1733202000536/Put/seqid=0 2024-12-03T05:00:01,558 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/B of 1fc1c9cce64fed6eabb56af3d145cb45 into 1ce8eb9fa7de4c4cbf1c8298af332ecc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:01,558 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T05:00:01,558 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/B, priority=13, startTime=1733202001483; duration=0sec 2024-12-03T05:00:01,558 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:01,558 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:B 2024-12-03T05:00:01,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/af92072c1ae14be49dc971b0a20c16dd is 50, key is test_row_0/A:col10/1733202000586/Put/seqid=0 2024-12-03T05:00:01,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741938_1114 (size=13425) 2024-12-03T05:00:01,588 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/f1eeaf9015934d74af25c4d4e2eb485d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f1eeaf9015934d74af25c4d4e2eb485d 2024-12-03T05:00:01,592 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-03T05:00:01,592 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:01,593 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-03T05:00:01,593 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:01,594 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-03T05:00:01,594 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:01,595 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-03T05:00:01,595 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:01,595 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fc1c9cce64fed6eabb56af3d145cb45/C of 1fc1c9cce64fed6eabb56af3d145cb45 into f1eeaf9015934d74af25c4d4e2eb485d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:01,595 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T05:00:01,596 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., storeName=1fc1c9cce64fed6eabb56af3d145cb45/C, priority=13, startTime=1733202001484; duration=0sec 2024-12-03T05:00:01,596 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:01,596 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fc1c9cce64fed6eabb56af3d145cb45:C 2024-12-03T05:00:01,598 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,598 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,598 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,598 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,598 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,598 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:01,598 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:01,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:01,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:01,599 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741939_1115 (size=12301) 2024-12-03T05:00:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T05:00:01,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. as already flushing 2024-12-03T05:00:01,613 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-03T05:00:01,613 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-03T05:00:01,613 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:01,613 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:01,613 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,613 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,613 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,613 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:01,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:01,623 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-03T05:00:01,623 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:01,623 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,623 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:01,742 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-03T05:00:01,742 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-03T05:00:01,743 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:01,743 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:01,743 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,743 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,743 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,743 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:01,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:01,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:00:02,002 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/af92072c1ae14be49dc971b0a20c16dd 2024-12-03T05:00:02,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/b8cccaac253046b3870f5f5dfebc5c20 is 50, key is test_row_0/B:col10/1733202000586/Put/seqid=0 2024-12-03T05:00:02,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741940_1116 (size=12301) 2024-12-03T05:00:02,444 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/b8cccaac253046b3870f5f5dfebc5c20 2024-12-03T05:00:02,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/847c5f698af248d8a72fe434ee31759f is 50, key is test_row_0/C:col10/1733202000586/Put/seqid=0 2024-12-03T05:00:02,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741941_1117 (size=12301) 2024-12-03T05:00:02,864 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/847c5f698af248d8a72fe434ee31759f 2024-12-03T05:00:02,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/af92072c1ae14be49dc971b0a20c16dd as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/af92072c1ae14be49dc971b0a20c16dd 2024-12-03T05:00:02,875 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/af92072c1ae14be49dc971b0a20c16dd, entries=150, sequenceid=493, filesize=12.0 K 2024-12-03T05:00:02,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/b8cccaac253046b3870f5f5dfebc5c20 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/b8cccaac253046b3870f5f5dfebc5c20 2024-12-03T05:00:02,883 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/b8cccaac253046b3870f5f5dfebc5c20, entries=150, sequenceid=493, filesize=12.0 K 2024-12-03T05:00:02,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/847c5f698af248d8a72fe434ee31759f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/847c5f698af248d8a72fe434ee31759f 2024-12-03T05:00:02,890 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/847c5f698af248d8a72fe434ee31759f, entries=150, sequenceid=493, filesize=12.0 K 2024-12-03T05:00:02,891 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=33.54 KB/34350 for 1fc1c9cce64fed6eabb56af3d145cb45 in 1340ms, sequenceid=493, compaction requested=false 2024-12-03T05:00:02,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 1fc1c9cce64fed6eabb56af3d145cb45: 2024-12-03T05:00:02,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:02,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-03T05:00:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-03T05:00:02,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-03T05:00:02,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1140 sec 2024-12-03T05:00:02,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.1220 sec 2024-12-03T05:00:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:00:02,931 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2544 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2536 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1201 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3601 rows 2024-12-03T05:00:02,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1199 2024-12-03T05:00:02,932 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3595 rows 2024-12-03T05:00:02,932 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:00:02,932 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:00:02,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:02,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:02,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:02,937 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:00:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:00:02,946 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202002946"}]},"ts":"1733202002946"} 2024-12-03T05:00:02,949 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:00:02,949 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:00:02,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:00:02,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fc1c9cce64fed6eabb56af3d145cb45, UNASSIGN}] 2024-12-03T05:00:02,957 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fc1c9cce64fed6eabb56af3d145cb45, UNASSIGN 2024-12-03T05:00:02,959 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1fc1c9cce64fed6eabb56af3d145cb45, regionState=CLOSING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:00:02,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fc1c9cce64fed6eabb56af3d145cb45, UNASSIGN because future has completed 2024-12-03T05:00:02,961 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:00:02,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:00:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:00:03,119 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T05:00:03,119 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:00:03,120 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 1fc1c9cce64fed6eabb56af3d145cb45, disabling compactions & flushes 2024-12-03T05:00:03,120 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:03,120 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:03,120 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. after waiting 0 ms 2024-12-03T05:00:03,120 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:03,120 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 1fc1c9cce64fed6eabb56af3d145cb45 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-03T05:00:03,125 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/f82b841b7db54bd295a204d627dfa050 is 50, key is test_row_0/A:col10/1733202001741/Put/seqid=0 2024-12-03T05:00:03,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741942_1118 (size=12301) 2024-12-03T05:00:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:00:03,531 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/f82b841b7db54bd295a204d627dfa050 2024-12-03T05:00:03,541 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/057cc9251cf848399120ddbad127dbf2 is 50, key is test_row_0/B:col10/1733202001741/Put/seqid=0 2024-12-03T05:00:03,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741943_1119 (size=12301) 2024-12-03T05:00:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:00:03,952 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/057cc9251cf848399120ddbad127dbf2 2024-12-03T05:00:03,961 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/f8e726cb222e47a0aec7436a12754e91 is 50, key is test_row_0/C:col10/1733202001741/Put/seqid=0 2024-12-03T05:00:03,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741944_1120 (size=12301) 2024-12-03T05:00:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:00:04,380 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/f8e726cb222e47a0aec7436a12754e91 2024-12-03T05:00:04,392 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/A/f82b841b7db54bd295a204d627dfa050 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/f82b841b7db54bd295a204d627dfa050 2024-12-03T05:00:04,397 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/f82b841b7db54bd295a204d627dfa050, entries=150, sequenceid=503, filesize=12.0 K 2024-12-03T05:00:04,398 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/B/057cc9251cf848399120ddbad127dbf2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/057cc9251cf848399120ddbad127dbf2 2024-12-03T05:00:04,403 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/057cc9251cf848399120ddbad127dbf2, entries=150, sequenceid=503, filesize=12.0 K 2024-12-03T05:00:04,404 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/.tmp/C/f8e726cb222e47a0aec7436a12754e91 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f8e726cb222e47a0aec7436a12754e91 2024-12-03T05:00:04,409 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f8e726cb222e47a0aec7436a12754e91, entries=150, sequenceid=503, filesize=12.0 K 2024-12-03T05:00:04,410 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1fc1c9cce64fed6eabb56af3d145cb45 in 1290ms, sequenceid=503, compaction requested=true 2024-12-03T05:00:04,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/3d7f0add986f4f69a33b4b6554c73172, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cfe53779d944f6abc414c01d9489682, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6b21e5248f6e499caf4ff6613f5af5ce, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/60df560727b24328824e916e59aba2d3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/52fce1d7ae04486997afa447d86f59ae, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/2ee0806acebf4cbf827e988ef8ee11cd, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/80e93d06b71742acbad42b17d5d36cf5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/2d6af0a72118497099eee4c18ce66aed, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cf04ff5ec89458faac67521bbf1c92b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ab507cd63ed04e15a896704d42370693, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7874ba34a55c496987e7754b784a0d57, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/4d3b43e664864dd19ec3f24dc2665201, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d7b43a6ec4694494a55471e537e9c658, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/002c9e44bb7e478e85a491c9a85a10d3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/5185627d54f54714acfeb43bf5f57818, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/05a0930c9a00455199dfc6f9df82d6a6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/033ddf637f89410da665bfbf6c69e04b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7c5dba6f424048ce9419f665246fe165, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6447ca6c63c24d969f061661ec5cabbc, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d9e086aa8c18441a9843bcd091828f8a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/a0aa15cbadab42e68e69c44b74752687, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/13deb3b63b554b0cb7a244f8edc8e688, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/75a1dc5687d34c02bb501e7de4c35e1e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/15b182a270f44eb5b6f19e076d9af67c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/40efe9a84e154c29a9f50525bb552b4b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ec4e43ce96fb441080e36d5656b2ec71, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/c466f8772fcc45c9b59d10e736720fbf, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7f4139cbcea842819b6bbb040e3c9c2c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/71bb8f4354284e0caa9d395da898c3c1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/1898fbcced10470788a2ec1620b4ab1c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8614891331be45929cf4b7c5f05a4634, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/f695307faf20433790a3d8a3c9920688, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d103dc9ab9674820acf582a6ab9b3475] to archive 2024-12-03T05:00:04,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:00:04,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/3d7f0add986f4f69a33b4b6554c73172 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/3d7f0add986f4f69a33b4b6554c73172 2024-12-03T05:00:04,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cfe53779d944f6abc414c01d9489682 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cfe53779d944f6abc414c01d9489682 2024-12-03T05:00:04,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6b21e5248f6e499caf4ff6613f5af5ce to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6b21e5248f6e499caf4ff6613f5af5ce 2024-12-03T05:00:04,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/60df560727b24328824e916e59aba2d3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/60df560727b24328824e916e59aba2d3 2024-12-03T05:00:04,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/52fce1d7ae04486997afa447d86f59ae to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/52fce1d7ae04486997afa447d86f59ae 2024-12-03T05:00:04,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/2ee0806acebf4cbf827e988ef8ee11cd to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/2ee0806acebf4cbf827e988ef8ee11cd 2024-12-03T05:00:04,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/80e93d06b71742acbad42b17d5d36cf5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/80e93d06b71742acbad42b17d5d36cf5 2024-12-03T05:00:04,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/2d6af0a72118497099eee4c18ce66aed to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/2d6af0a72118497099eee4c18ce66aed 2024-12-03T05:00:04,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cf04ff5ec89458faac67521bbf1c92b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8cf04ff5ec89458faac67521bbf1c92b 2024-12-03T05:00:04,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ab507cd63ed04e15a896704d42370693 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ab507cd63ed04e15a896704d42370693 2024-12-03T05:00:04,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7874ba34a55c496987e7754b784a0d57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7874ba34a55c496987e7754b784a0d57 2024-12-03T05:00:04,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/4d3b43e664864dd19ec3f24dc2665201 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/4d3b43e664864dd19ec3f24dc2665201 2024-12-03T05:00:04,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d7b43a6ec4694494a55471e537e9c658 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d7b43a6ec4694494a55471e537e9c658 2024-12-03T05:00:04,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/002c9e44bb7e478e85a491c9a85a10d3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/002c9e44bb7e478e85a491c9a85a10d3 2024-12-03T05:00:04,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/5185627d54f54714acfeb43bf5f57818 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/5185627d54f54714acfeb43bf5f57818 2024-12-03T05:00:04,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/05a0930c9a00455199dfc6f9df82d6a6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/05a0930c9a00455199dfc6f9df82d6a6 2024-12-03T05:00:04,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/033ddf637f89410da665bfbf6c69e04b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/033ddf637f89410da665bfbf6c69e04b 2024-12-03T05:00:04,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7c5dba6f424048ce9419f665246fe165 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7c5dba6f424048ce9419f665246fe165 2024-12-03T05:00:04,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6447ca6c63c24d969f061661ec5cabbc to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/6447ca6c63c24d969f061661ec5cabbc 2024-12-03T05:00:04,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d9e086aa8c18441a9843bcd091828f8a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d9e086aa8c18441a9843bcd091828f8a 2024-12-03T05:00:04,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/a0aa15cbadab42e68e69c44b74752687 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/a0aa15cbadab42e68e69c44b74752687 2024-12-03T05:00:04,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/13deb3b63b554b0cb7a244f8edc8e688 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/13deb3b63b554b0cb7a244f8edc8e688 2024-12-03T05:00:04,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/75a1dc5687d34c02bb501e7de4c35e1e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/75a1dc5687d34c02bb501e7de4c35e1e 2024-12-03T05:00:04,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/15b182a270f44eb5b6f19e076d9af67c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/15b182a270f44eb5b6f19e076d9af67c 2024-12-03T05:00:04,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/40efe9a84e154c29a9f50525bb552b4b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/40efe9a84e154c29a9f50525bb552b4b 2024-12-03T05:00:04,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ec4e43ce96fb441080e36d5656b2ec71 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/ec4e43ce96fb441080e36d5656b2ec71 2024-12-03T05:00:04,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/c466f8772fcc45c9b59d10e736720fbf to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/c466f8772fcc45c9b59d10e736720fbf 2024-12-03T05:00:04,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7f4139cbcea842819b6bbb040e3c9c2c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/7f4139cbcea842819b6bbb040e3c9c2c 2024-12-03T05:00:04,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/71bb8f4354284e0caa9d395da898c3c1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/71bb8f4354284e0caa9d395da898c3c1 2024-12-03T05:00:04,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/1898fbcced10470788a2ec1620b4ab1c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/1898fbcced10470788a2ec1620b4ab1c 2024-12-03T05:00:04,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8614891331be45929cf4b7c5f05a4634 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/8614891331be45929cf4b7c5f05a4634 2024-12-03T05:00:04,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/f695307faf20433790a3d8a3c9920688 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/f695307faf20433790a3d8a3c9920688 2024-12-03T05:00:04,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d103dc9ab9674820acf582a6ab9b3475 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/d103dc9ab9674820acf582a6ab9b3475 2024-12-03T05:00:04,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f0def0871b7443279c90cf0f9301acd1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/208193c6a60348659c942e58c96996b4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a48473c0360a4e09917f3aad40e74808, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e4030eef25c848a98d98da964390508a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/0ed2391fa37749d28452ef02bef5c28c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/76145898de99419b83338e33afacf327, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/6b188b3e5a0946b88944c225550f7f03, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d32fb950059646f2a023b94fc388b7e7, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e4c3d2f401804c84a8b8e95d56804007, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f79d79aa2fe94d0e88ea3fdcbe84c8ef, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a8dfc54b850945dba325072f77f5caa2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e5190e3f90da4107bca15862a16a7d64, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c7313a8f9b5b486aac20b4955303c80b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/678ca55d873a46578f77869fe3c1f5e0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f681de595bc94c4ba0fae59d33b788e5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/9b39e91e24f74b51b4ebba013a464059, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d53405749e2740e6894fd41b963a121d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/05cac77ed7264ebeacb9dcc0b2ab4094, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5bf923e076fe47b2b4066f93c5df9514, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d9f4fcb7a24d4c74a1c1946f877e38c2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/84eccff191aa4cfe82f8ea2c1bdc0301, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/ddfe2e577cd04f8cab06c5ae67e75027, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d5cb3310055b46618cbfd328aa4be79b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/acc4c91ce97e4fd3a16c008a40540e39, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c64c969879864d99bbfc1983bf513d87, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/8386188eac5d41b2a02b78ecb89fce3a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/962ce83e90284ecd97204b1769ef9854, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5fb3d9a9d0814e11aa42047ff048e89f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/aa63e8ed9ac94e10bfb58e12fc882831, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/109575e3153e4bada1e41a9cd6cbad45, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/09b36222df784782a0334aacd2da1d7f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/33a92d9a6e6f400cb1f76407943655a3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c5ea719c8e3e4a66bae20b3028915d6c] to archive 2024-12-03T05:00:04,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:00:04,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f0def0871b7443279c90cf0f9301acd1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f0def0871b7443279c90cf0f9301acd1 2024-12-03T05:00:04,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/208193c6a60348659c942e58c96996b4 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/208193c6a60348659c942e58c96996b4 2024-12-03T05:00:04,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a48473c0360a4e09917f3aad40e74808 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a48473c0360a4e09917f3aad40e74808 2024-12-03T05:00:04,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e4030eef25c848a98d98da964390508a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e4030eef25c848a98d98da964390508a 2024-12-03T05:00:04,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/0ed2391fa37749d28452ef02bef5c28c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/0ed2391fa37749d28452ef02bef5c28c 2024-12-03T05:00:04,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/76145898de99419b83338e33afacf327 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/76145898de99419b83338e33afacf327 2024-12-03T05:00:04,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/6b188b3e5a0946b88944c225550f7f03 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/6b188b3e5a0946b88944c225550f7f03 2024-12-03T05:00:04,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d32fb950059646f2a023b94fc388b7e7 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d32fb950059646f2a023b94fc388b7e7 2024-12-03T05:00:04,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e4c3d2f401804c84a8b8e95d56804007 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e4c3d2f401804c84a8b8e95d56804007 2024-12-03T05:00:04,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f79d79aa2fe94d0e88ea3fdcbe84c8ef to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f79d79aa2fe94d0e88ea3fdcbe84c8ef 2024-12-03T05:00:04,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a8dfc54b850945dba325072f77f5caa2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/a8dfc54b850945dba325072f77f5caa2 2024-12-03T05:00:04,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e5190e3f90da4107bca15862a16a7d64 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/e5190e3f90da4107bca15862a16a7d64 2024-12-03T05:00:04,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c7313a8f9b5b486aac20b4955303c80b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c7313a8f9b5b486aac20b4955303c80b 2024-12-03T05:00:04,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/678ca55d873a46578f77869fe3c1f5e0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/678ca55d873a46578f77869fe3c1f5e0 2024-12-03T05:00:04,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f681de595bc94c4ba0fae59d33b788e5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/f681de595bc94c4ba0fae59d33b788e5 2024-12-03T05:00:04,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/9b39e91e24f74b51b4ebba013a464059 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/9b39e91e24f74b51b4ebba013a464059 2024-12-03T05:00:04,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d53405749e2740e6894fd41b963a121d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d53405749e2740e6894fd41b963a121d 2024-12-03T05:00:04,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/05cac77ed7264ebeacb9dcc0b2ab4094 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/05cac77ed7264ebeacb9dcc0b2ab4094 2024-12-03T05:00:04,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5bf923e076fe47b2b4066f93c5df9514 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5bf923e076fe47b2b4066f93c5df9514 2024-12-03T05:00:04,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d9f4fcb7a24d4c74a1c1946f877e38c2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d9f4fcb7a24d4c74a1c1946f877e38c2 2024-12-03T05:00:04,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/84eccff191aa4cfe82f8ea2c1bdc0301 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/84eccff191aa4cfe82f8ea2c1bdc0301 2024-12-03T05:00:04,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/ddfe2e577cd04f8cab06c5ae67e75027 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/ddfe2e577cd04f8cab06c5ae67e75027 2024-12-03T05:00:04,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d5cb3310055b46618cbfd328aa4be79b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/d5cb3310055b46618cbfd328aa4be79b 2024-12-03T05:00:04,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/acc4c91ce97e4fd3a16c008a40540e39 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/acc4c91ce97e4fd3a16c008a40540e39 2024-12-03T05:00:04,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c64c969879864d99bbfc1983bf513d87 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c64c969879864d99bbfc1983bf513d87 2024-12-03T05:00:04,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/8386188eac5d41b2a02b78ecb89fce3a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/8386188eac5d41b2a02b78ecb89fce3a 2024-12-03T05:00:04,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/962ce83e90284ecd97204b1769ef9854 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/962ce83e90284ecd97204b1769ef9854 2024-12-03T05:00:04,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5fb3d9a9d0814e11aa42047ff048e89f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/5fb3d9a9d0814e11aa42047ff048e89f 2024-12-03T05:00:04,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/aa63e8ed9ac94e10bfb58e12fc882831 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/aa63e8ed9ac94e10bfb58e12fc882831 2024-12-03T05:00:04,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/109575e3153e4bada1e41a9cd6cbad45 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/109575e3153e4bada1e41a9cd6cbad45 2024-12-03T05:00:04,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/09b36222df784782a0334aacd2da1d7f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/09b36222df784782a0334aacd2da1d7f 2024-12-03T05:00:04,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/33a92d9a6e6f400cb1f76407943655a3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/33a92d9a6e6f400cb1f76407943655a3 2024-12-03T05:00:04,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c5ea719c8e3e4a66bae20b3028915d6c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/c5ea719c8e3e4a66bae20b3028915d6c 2024-12-03T05:00:04,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/5f08f6f99b7d448ab8709e186d09d5bf, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/ef6e7c3c5b5747dda3e17ef837158694, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/e751cd436fee4403b5a18168d1cb8700, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/9a5a038d305f4f87acd61f58e4e9b2ae, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0013f32453354c44949dd48ead1bc28d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/51e7f76fa5d740fb92f8098cddca2786, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/930f2bdf7b5f45df89d82dd6f23cf515, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0bb4b78f84da4cbea7344c38ccbf08a6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/756243fb555d43ac83790792da083105, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/621316aff8134347ba541161937c5be1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/4a750660366f415686db1e695d012850, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/969babe6b1534388bf89199109158ded, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/79b4a16e91114352a8ff90aa6e6346e4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/42470e497df14ba0afbe258bb881c1d0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1d55e16503be4fa2ba4c6d351073e94b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/94d09ec6438241e787c192731266afd5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2c166ed184884e8f9a0c689fb5e957ce, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1647dc26dfa14007ba1dfd823823fd36, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/6899fdf3eb93487c8804e14751968515, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1cd06ed39268416fbe70751af0f564c9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/7b60682c706c471c930932b27366a2d2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2f46739e5dfa4c33a5ad94e175c27ee7, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/fbb792de09bd43fdacaf0b2b003de00f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/31af8f78c4284241a985339fab9a63ca, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/b394b2286ddc41a3b5dd76cbe17d9af9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f21cb5a29fe54834b35b9fb4015dbe2a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/339572eca14b422b80ea125ed1bba32c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/fc04e46c3e3a4722be1dc43ebb61f8c8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f87339b0955e4e0ea67d8804d158a046, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/868a78befab6473cb2018664d96f1611, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1ae0b7955eeb463bbddb806c4c98beb0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/80b36128eb324bd5adbb6e67395d269c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/9945000a685b4e25a1ba90d6cd684094] to archive 2024-12-03T05:00:04,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:00:04,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/5f08f6f99b7d448ab8709e186d09d5bf to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/5f08f6f99b7d448ab8709e186d09d5bf 2024-12-03T05:00:04,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/ef6e7c3c5b5747dda3e17ef837158694 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/ef6e7c3c5b5747dda3e17ef837158694 2024-12-03T05:00:04,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/e751cd436fee4403b5a18168d1cb8700 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/e751cd436fee4403b5a18168d1cb8700 2024-12-03T05:00:04,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/9a5a038d305f4f87acd61f58e4e9b2ae to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/9a5a038d305f4f87acd61f58e4e9b2ae 2024-12-03T05:00:04,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0013f32453354c44949dd48ead1bc28d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0013f32453354c44949dd48ead1bc28d 2024-12-03T05:00:04,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/51e7f76fa5d740fb92f8098cddca2786 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/51e7f76fa5d740fb92f8098cddca2786 2024-12-03T05:00:04,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/930f2bdf7b5f45df89d82dd6f23cf515 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/930f2bdf7b5f45df89d82dd6f23cf515 2024-12-03T05:00:04,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0bb4b78f84da4cbea7344c38ccbf08a6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/0bb4b78f84da4cbea7344c38ccbf08a6 2024-12-03T05:00:04,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/756243fb555d43ac83790792da083105 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/756243fb555d43ac83790792da083105 2024-12-03T05:00:04,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/621316aff8134347ba541161937c5be1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/621316aff8134347ba541161937c5be1 2024-12-03T05:00:04,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/4a750660366f415686db1e695d012850 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/4a750660366f415686db1e695d012850 2024-12-03T05:00:04,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/969babe6b1534388bf89199109158ded to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/969babe6b1534388bf89199109158ded 2024-12-03T05:00:04,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/79b4a16e91114352a8ff90aa6e6346e4 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/79b4a16e91114352a8ff90aa6e6346e4 2024-12-03T05:00:04,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/42470e497df14ba0afbe258bb881c1d0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/42470e497df14ba0afbe258bb881c1d0 2024-12-03T05:00:04,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1d55e16503be4fa2ba4c6d351073e94b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1d55e16503be4fa2ba4c6d351073e94b 2024-12-03T05:00:04,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/94d09ec6438241e787c192731266afd5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/94d09ec6438241e787c192731266afd5 2024-12-03T05:00:04,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2c166ed184884e8f9a0c689fb5e957ce to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2c166ed184884e8f9a0c689fb5e957ce 2024-12-03T05:00:04,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1647dc26dfa14007ba1dfd823823fd36 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1647dc26dfa14007ba1dfd823823fd36 2024-12-03T05:00:04,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/6899fdf3eb93487c8804e14751968515 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/6899fdf3eb93487c8804e14751968515 2024-12-03T05:00:04,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1cd06ed39268416fbe70751af0f564c9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1cd06ed39268416fbe70751af0f564c9 2024-12-03T05:00:04,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/7b60682c706c471c930932b27366a2d2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/7b60682c706c471c930932b27366a2d2 2024-12-03T05:00:04,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2f46739e5dfa4c33a5ad94e175c27ee7 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/2f46739e5dfa4c33a5ad94e175c27ee7 2024-12-03T05:00:04,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/fbb792de09bd43fdacaf0b2b003de00f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/fbb792de09bd43fdacaf0b2b003de00f 2024-12-03T05:00:04,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/31af8f78c4284241a985339fab9a63ca to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/31af8f78c4284241a985339fab9a63ca 2024-12-03T05:00:04,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/b394b2286ddc41a3b5dd76cbe17d9af9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/b394b2286ddc41a3b5dd76cbe17d9af9 2024-12-03T05:00:04,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f21cb5a29fe54834b35b9fb4015dbe2a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f21cb5a29fe54834b35b9fb4015dbe2a 2024-12-03T05:00:04,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/339572eca14b422b80ea125ed1bba32c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/339572eca14b422b80ea125ed1bba32c 2024-12-03T05:00:04,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/fc04e46c3e3a4722be1dc43ebb61f8c8 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/fc04e46c3e3a4722be1dc43ebb61f8c8 2024-12-03T05:00:04,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f87339b0955e4e0ea67d8804d158a046 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f87339b0955e4e0ea67d8804d158a046 2024-12-03T05:00:04,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/868a78befab6473cb2018664d96f1611 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/868a78befab6473cb2018664d96f1611 2024-12-03T05:00:04,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1ae0b7955eeb463bbddb806c4c98beb0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/1ae0b7955eeb463bbddb806c4c98beb0 2024-12-03T05:00:04,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/80b36128eb324bd5adbb6e67395d269c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/80b36128eb324bd5adbb6e67395d269c 2024-12-03T05:00:04,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/9945000a685b4e25a1ba90d6cd684094 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/9945000a685b4e25a1ba90d6cd684094 2024-12-03T05:00:04,621 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/recovered.edits/506.seqid, newMaxSeqId=506, maxSeqId=1 2024-12-03T05:00:04,622 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. 2024-12-03T05:00:04,622 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 1fc1c9cce64fed6eabb56af3d145cb45: Waiting for close lock at 1733202003120Running coprocessor pre-close hooks at 1733202003120Disabling compacts and flushes for region at 1733202003120Disabling writes for close at 1733202003120Obtaining lock to block concurrent updates at 1733202003121 (+1 ms)Preparing flush snapshotting stores in 1fc1c9cce64fed6eabb56af3d145cb45 at 1733202003121Finished memstore snapshotting TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733202003121Flushing stores of TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45. at 1733202003122 (+1 ms)Flushing 1fc1c9cce64fed6eabb56af3d145cb45/A: creating writer at 1733202003122Flushing 1fc1c9cce64fed6eabb56af3d145cb45/A: appending metadata at 1733202003125 (+3 ms)Flushing 1fc1c9cce64fed6eabb56af3d145cb45/A: closing flushed file at 1733202003125Flushing 1fc1c9cce64fed6eabb56af3d145cb45/B: creating writer at 1733202003537 (+412 ms)Flushing 1fc1c9cce64fed6eabb56af3d145cb45/B: appending metadata at 1733202003540 (+3 ms)Flushing 1fc1c9cce64fed6eabb56af3d145cb45/B: closing flushed file at 1733202003540Flushing 1fc1c9cce64fed6eabb56af3d145cb45/C: creating writer at 1733202003957 (+417 ms)Flushing 1fc1c9cce64fed6eabb56af3d145cb45/C: appending metadata at 1733202003960 (+3 ms)Flushing 1fc1c9cce64fed6eabb56af3d145cb45/C: closing flushed file at 1733202003960Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32c55490: reopening flushed file at 1733202004391 (+431 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bf93ca7: reopening flushed file at 1733202004397 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@315160ad: reopening flushed file at 1733202004403 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1fc1c9cce64fed6eabb56af3d145cb45 in 1290ms, sequenceid=503, compaction requested=true at 1733202004410 (+7 ms)Writing region close event to WAL at 1733202004617 (+207 ms)Running coprocessor post-close hooks at 1733202004621 (+4 ms)Closed at 1733202004621 2024-12-03T05:00:04,625 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T05:00:04,625 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1fc1c9cce64fed6eabb56af3d145cb45, regionState=CLOSED 2024-12-03T05:00:04,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:00:04,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T05:00:04,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 1fc1c9cce64fed6eabb56af3d145cb45, server=32beccd76b35,34719,1733201977102 in 1.6680 sec 2024-12-03T05:00:04,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-12-03T05:00:04,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fc1c9cce64fed6eabb56af3d145cb45, UNASSIGN in 1.6750 sec 2024-12-03T05:00:04,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T05:00:04,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6830 sec 2024-12-03T05:00:04,638 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202004638"}]},"ts":"1733202004638"} 2024-12-03T05:00:04,640 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T05:00:04,640 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T05:00:04,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7020 sec 2024-12-03T05:00:04,931 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:00:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:00:05,101 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:05,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T05:00:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:05,112 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T05:00:05,114 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:05,127 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T05:00:05,135 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/recovered.edits] 2024-12-03T05:00:05,183 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/369e6f72d9824c27b82b06789de17c60 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/369e6f72d9824c27b82b06789de17c60 2024-12-03T05:00:05,203 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/af92072c1ae14be49dc971b0a20c16dd to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/af92072c1ae14be49dc971b0a20c16dd 2024-12-03T05:00:05,206 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/f82b841b7db54bd295a204d627dfa050 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/A/f82b841b7db54bd295a204d627dfa050 2024-12-03T05:00:05,210 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/057cc9251cf848399120ddbad127dbf2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/057cc9251cf848399120ddbad127dbf2 2024-12-03T05:00:05,215 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/1ce8eb9fa7de4c4cbf1c8298af332ecc to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/1ce8eb9fa7de4c4cbf1c8298af332ecc 2024-12-03T05:00:05,218 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/b8cccaac253046b3870f5f5dfebc5c20 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/B/b8cccaac253046b3870f5f5dfebc5c20 2024-12-03T05:00:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T05:00:05,227 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/847c5f698af248d8a72fe434ee31759f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/847c5f698af248d8a72fe434ee31759f 2024-12-03T05:00:05,233 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f1eeaf9015934d74af25c4d4e2eb485d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f1eeaf9015934d74af25c4d4e2eb485d 2024-12-03T05:00:05,240 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f8e726cb222e47a0aec7436a12754e91 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/C/f8e726cb222e47a0aec7436a12754e91 2024-12-03T05:00:05,249 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/recovered.edits/506.seqid to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45/recovered.edits/506.seqid 2024-12-03T05:00:05,251 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1fc1c9cce64fed6eabb56af3d145cb45 2024-12-03T05:00:05,253 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T05:00:05,273 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:05,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-03T05:00:05,295 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T05:00:05,338 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T05:00:05,341 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:05,341 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T05:00:05,341 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733202005341"}]},"ts":"9223372036854775807"} 2024-12-03T05:00:05,347 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T05:00:05,347 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1fc1c9cce64fed6eabb56af3d145cb45, NAME => 'TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T05:00:05,347 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T05:00:05,348 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733202005347"}]},"ts":"9223372036854775807"} 2024-12-03T05:00:05,352 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T05:00:05,353 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:05,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 247 msec 2024-12-03T05:00:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T05:00:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T05:00:05,432 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:05,444 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;32beccd76b35:34719-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1161058445_22 at /127.0.0.1:38000 [Waiting for operation #352] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-775871842_22 at /127.0.0.1:60774 [Waiting for operation #331] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/32beccd76b35:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=912 (was 629) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3519 (was 4524) 2024-12-03T05:00:05,453 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=912, ProcessCount=11, AvailableMemoryMB=3518 2024-12-03T05:00:05,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:00:05,455 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:00:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:05,458 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:00:05,458 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:05,458 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-12-03T05:00:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T05:00:05,459 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:00:05,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741945_1121 (size=959) 2024-12-03T05:00:05,468 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d 2024-12-03T05:00:05,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741946_1122 (size=53) 2024-12-03T05:00:05,480 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:00:05,480 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b4ec74c11861541ec3b2d9523601012e, disabling compactions & flushes 2024-12-03T05:00:05,480 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:05,480 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:05,480 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. after waiting 0 ms 2024-12-03T05:00:05,480 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:05,480 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:05,480 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b4ec74c11861541ec3b2d9523601012e: Waiting for close lock at 1733202005480Disabling compacts and flushes for region at 1733202005480Disabling writes for close at 1733202005480Writing region close event to WAL at 1733202005480Closed at 1733202005480 2024-12-03T05:00:05,482 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:00:05,482 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733202005482"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733202005482"}]},"ts":"1733202005482"} 2024-12-03T05:00:05,484 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:00:05,486 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:00:05,486 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202005486"}]},"ts":"1733202005486"} 2024-12-03T05:00:05,488 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T05:00:05,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, ASSIGN}] 2024-12-03T05:00:05,489 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, ASSIGN 2024-12-03T05:00:05,490 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, ASSIGN; state=OFFLINE, location=32beccd76b35,34719,1733201977102; forceNewPlan=false, retain=false 2024-12-03T05:00:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T05:00:05,641 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b4ec74c11861541ec3b2d9523601012e, regionState=OPENING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:00:05,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, ASSIGN because future has completed 2024-12-03T05:00:05,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:00:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T05:00:05,801 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:05,802 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:00:05,802 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,802 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:00:05,802 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,802 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,804 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,807 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4ec74c11861541ec3b2d9523601012e columnFamilyName A 2024-12-03T05:00:05,807 DEBUG [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:05,808 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(327): Store=b4ec74c11861541ec3b2d9523601012e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:00:05,808 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,810 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4ec74c11861541ec3b2d9523601012e columnFamilyName B 2024-12-03T05:00:05,810 DEBUG [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:05,811 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(327): Store=b4ec74c11861541ec3b2d9523601012e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:00:05,811 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,813 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4ec74c11861541ec3b2d9523601012e columnFamilyName C 2024-12-03T05:00:05,813 DEBUG [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:05,814 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(327): Store=b4ec74c11861541ec3b2d9523601012e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:00:05,815 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,815 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,820 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,821 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,821 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,822 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:00:05,824 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,830 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:00:05,830 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened b4ec74c11861541ec3b2d9523601012e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66377644, jitterRate=-0.010896027088165283}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:00:05,830 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:05,831 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for b4ec74c11861541ec3b2d9523601012e: Running coprocessor pre-open hook at 1733202005803Writing region info on filesystem at 1733202005803Initializing all the Stores at 1733202005803Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202005803Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202005804 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202005804Cleaning up temporary data from old regions at 1733202005821 (+17 ms)Running coprocessor post-open hooks at 1733202005830 (+9 ms)Region opened successfully at 1733202005831 (+1 ms) 2024-12-03T05:00:05,832 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., pid=30, masterSystemTime=1733202005797 2024-12-03T05:00:05,836 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:05,836 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:05,836 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b4ec74c11861541ec3b2d9523601012e, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:00:05,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:00:05,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T05:00:05,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 in 198 msec 2024-12-03T05:00:05,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-03T05:00:05,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, ASSIGN in 358 msec 2024-12-03T05:00:05,851 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:00:05,852 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202005852"}]},"ts":"1733202005852"} 2024-12-03T05:00:05,854 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T05:00:05,855 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:00:05,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 400 msec 2024-12-03T05:00:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T05:00:06,091 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:06,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:06,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:06,095 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388a3adb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:06,097 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f1990d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,099 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:06,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:06,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:06,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:06,104 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:00:06,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,43423,1733201976038 2024-12-03T05:00:06,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@695273e8 2024-12-03T05:00:06,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:00:06,107 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:00:06,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:00:06,114 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:00:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:06,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741947_1123 (size=995) 2024-12-03T05:00:06,157 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-03T05:00:06,157 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-03T05:00:06,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:00:06,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, REOPEN/MOVE}] 2024-12-03T05:00:06,171 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, REOPEN/MOVE 2024-12-03T05:00:06,172 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b4ec74c11861541ec3b2d9523601012e, regionState=CLOSING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:00:06,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, REOPEN/MOVE because future has completed 2024-12-03T05:00:06,175 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:00:06,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:00:06,328 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,328 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:00:06,328 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing b4ec74c11861541ec3b2d9523601012e, disabling compactions & flushes 2024-12-03T05:00:06,328 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:06,329 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:06,329 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. after waiting 0 ms 2024-12-03T05:00:06,329 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:06,334 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T05:00:06,334 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:06,335 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for b4ec74c11861541ec3b2d9523601012e: Waiting for close lock at 1733202006328Running coprocessor pre-close hooks at 1733202006328Disabling compacts and flushes for region at 1733202006328Disabling writes for close at 1733202006329 (+1 ms)Writing region close event to WAL at 1733202006329Running coprocessor post-close hooks at 1733202006334 (+5 ms)Closed at 1733202006334 2024-12-03T05:00:06,335 WARN [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: b4ec74c11861541ec3b2d9523601012e to self. 2024-12-03T05:00:06,338 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,338 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b4ec74c11861541ec3b2d9523601012e, regionState=CLOSED 2024-12-03T05:00:06,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:00:06,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-03T05:00:06,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 in 167 msec 2024-12-03T05:00:06,345 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, REOPEN/MOVE; state=CLOSED, location=32beccd76b35,34719,1733201977102; forceNewPlan=false, retain=true 2024-12-03T05:00:06,496 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b4ec74c11861541ec3b2d9523601012e, regionState=OPENING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:00:06,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, REOPEN/MOVE because future has completed 2024-12-03T05:00:06,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:00:06,655 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:06,655 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:00:06,656 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,656 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:00:06,656 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,656 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,665 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,672 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4ec74c11861541ec3b2d9523601012e columnFamilyName A 2024-12-03T05:00:06,676 DEBUG [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:06,677 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(327): Store=b4ec74c11861541ec3b2d9523601012e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:00:06,678 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,680 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4ec74c11861541ec3b2d9523601012e columnFamilyName B 2024-12-03T05:00:06,680 DEBUG [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:06,681 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(327): Store=b4ec74c11861541ec3b2d9523601012e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:00:06,681 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,682 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4ec74c11861541ec3b2d9523601012e columnFamilyName C 2024-12-03T05:00:06,682 DEBUG [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:06,682 INFO [StoreOpener-b4ec74c11861541ec3b2d9523601012e-1 {}] regionserver.HStore(327): Store=b4ec74c11861541ec3b2d9523601012e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:00:06,682 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,683 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,685 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T05:00:06,690 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,690 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,690 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:00:06,692 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,694 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened b4ec74c11861541ec3b2d9523601012e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61621840, jitterRate=-0.08176302909851074}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:00:06,694 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,695 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for b4ec74c11861541ec3b2d9523601012e: Running coprocessor pre-open hook at 1733202006656Writing region info on filesystem at 1733202006656Initializing all the Stores at 1733202006657 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202006657Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202006661 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202006661Cleaning up temporary data from old regions at 1733202006690 (+29 ms)Running coprocessor post-open hooks at 1733202006694 (+4 ms)Region opened successfully at 1733202006695 (+1 ms) 2024-12-03T05:00:06,696 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., pid=35, masterSystemTime=1733202006651 2024-12-03T05:00:06,700 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b4ec74c11861541ec3b2d9523601012e, regionState=OPEN, openSeqNum=5, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:00:06,701 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:06,701 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:06,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:00:06,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-12-03T05:00:06,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 in 206 msec 2024-12-03T05:00:06,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-03T05:00:06,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, REOPEN/MOVE in 537 msec 2024-12-03T05:00:06,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T05:00:06,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 554 msec 2024-12-03T05:00:06,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 603 msec 2024-12-03T05:00:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T05:00:06,724 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:06,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:06,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:06,726 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3898cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:06,729 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d767205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:06,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:06,735 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ee2807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:06,737 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c35db67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:06,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:06,742 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6ea8c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:06,744 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0c7109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:06,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:06,747 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6fa763, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:06,749 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a95511e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,753 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:06,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:06,754 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5241e4a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:06,756 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b30d0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:06,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:06,758 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e70b52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:06,760 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bcc236c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:06,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:06,765 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51784511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:06,766 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7a6519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:06,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:06,768 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:06,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:06,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d661940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:06,770 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:06,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6739a3af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:06,771 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:06,772 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4783af9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:06,774 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1358217d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:06,776 DEBUG [Thread-631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:06,776 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:06,776 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-03T05:00:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:06,777 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:06,778 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:06,779 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:06,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:00:06,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:06,779 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:06,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:06,780 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:06,780 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:06,781 DEBUG [Thread-622 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:06,781 DEBUG [Thread-622 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:06,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:06,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:06,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:06,782 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:06,783 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:06,784 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:06,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:00:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:00:06,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:00:06,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:06,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:00:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:00:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:00:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:00:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:00:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:00:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:06,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:00:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:00:06,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:00:06,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:06,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:06,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202066930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:06,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202066930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:06,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036d6c9d6d268042aba0fb2a634b4215c5_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202006862/Put/seqid=0 2024-12-03T05:00:06,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741948_1124 (size=17034) 2024-12-03T05:00:06,957 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:06,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202066939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:06,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202066939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:06,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202066940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:06,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:06,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:06,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:06,963 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036d6c9d6d268042aba0fb2a634b4215c5_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036d6c9d6d268042aba0fb2a634b4215c5_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:06,966 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/d32c093578d94c24bba02c5562d91a9d, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:06,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/d32c093578d94c24bba02c5562d91a9d is 175, key is test_row_0/A:col10/1733202006862/Put/seqid=0 2024-12-03T05:00:07,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741949_1125 (size=48139) 2024-12-03T05:00:07,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202067051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202067051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202067060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202067070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202067071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:00:07,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:07,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:07,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:07,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:00:07,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:00:07,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:07,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:07,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:07,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202067262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202067262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202067281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202067290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202067290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:00:07,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:07,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:07,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:07,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:00:07,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/d32c093578d94c24bba02c5562d91a9d 2024-12-03T05:00:07,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/21206a532e384a1aabc18ba1876f6b34 is 50, key is test_row_0/B:col10/1733202006862/Put/seqid=0 2024-12-03T05:00:07,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741950_1126 (size=12001) 2024-12-03T05:00:07,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/21206a532e384a1aabc18ba1876f6b34 2024-12-03T05:00:07,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:00:07,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:07,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:07,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:07,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:07,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202067571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202067571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/487620b102df4aa2be2b0f57011cae13 is 50, key is test_row_0/C:col10/1733202006862/Put/seqid=0 2024-12-03T05:00:07,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202067590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202067601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202067601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:07,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:07,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741951_1127 (size=12001) 2024-12-03T05:00:07,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/487620b102df4aa2be2b0f57011cae13 2024-12-03T05:00:07,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/d32c093578d94c24bba02c5562d91a9d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d32c093578d94c24bba02c5562d91a9d 2024-12-03T05:00:07,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d32c093578d94c24bba02c5562d91a9d, entries=250, sequenceid=17, filesize=47.0 K 2024-12-03T05:00:07,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/21206a532e384a1aabc18ba1876f6b34 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/21206a532e384a1aabc18ba1876f6b34 2024-12-03T05:00:07,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/21206a532e384a1aabc18ba1876f6b34, entries=150, sequenceid=17, filesize=11.7 K 2024-12-03T05:00:07,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/487620b102df4aa2be2b0f57011cae13 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/487620b102df4aa2be2b0f57011cae13 2024-12-03T05:00:07,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/487620b102df4aa2be2b0f57011cae13, entries=150, sequenceid=17, filesize=11.7 K 2024-12-03T05:00:07,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for b4ec74c11861541ec3b2d9523601012e in 807ms, sequenceid=17, compaction requested=false 2024-12-03T05:00:07,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:07,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:00:07,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:07,714 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:00:07,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203420e30552ca34e608f7f665bee7edd23_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202006926/Put/seqid=0 2024-12-03T05:00:07,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741952_1128 (size=12154) 2024-12-03T05:00:07,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:07,820 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203420e30552ca34e608f7f665bee7edd23_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203420e30552ca34e608f7f665bee7edd23_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:07,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/d2f9ad92af084427a6773ac067f70fea, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:07,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/d2f9ad92af084427a6773ac067f70fea is 175, key is test_row_0/A:col10/1733202006926/Put/seqid=0 2024-12-03T05:00:07,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741953_1129 (size=30955) 2024-12-03T05:00:07,852 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/d2f9ad92af084427a6773ac067f70fea 2024-12-03T05:00:07,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/464090caf3fe4e279c08c00826594c19 is 50, key is test_row_0/B:col10/1733202006926/Put/seqid=0 2024-12-03T05:00:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:00:07,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741954_1130 (size=12001) 2024-12-03T05:00:08,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:08,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202068111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202068121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202068121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202068123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202068125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202068230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202068241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202068244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,344 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/464090caf3fe4e279c08c00826594c19 2024-12-03T05:00:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/ada920a11ae54afcb3f6fe933f9fed18 is 50, key is test_row_0/C:col10/1733202006926/Put/seqid=0 2024-12-03T05:00:08,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741955_1131 (size=12001) 2024-12-03T05:00:08,402 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/ada920a11ae54afcb3f6fe933f9fed18 2024-12-03T05:00:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/d2f9ad92af084427a6773ac067f70fea as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d2f9ad92af084427a6773ac067f70fea 2024-12-03T05:00:08,417 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d2f9ad92af084427a6773ac067f70fea, entries=150, sequenceid=42, filesize=30.2 K 2024-12-03T05:00:08,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/464090caf3fe4e279c08c00826594c19 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/464090caf3fe4e279c08c00826594c19 2024-12-03T05:00:08,424 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/464090caf3fe4e279c08c00826594c19, entries=150, sequenceid=42, filesize=11.7 K 2024-12-03T05:00:08,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/ada920a11ae54afcb3f6fe933f9fed18 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ada920a11ae54afcb3f6fe933f9fed18 2024-12-03T05:00:08,435 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ada920a11ae54afcb3f6fe933f9fed18, entries=150, sequenceid=42, filesize=11.7 K 2024-12-03T05:00:08,439 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for b4ec74c11861541ec3b2d9523601012e in 725ms, sequenceid=42, compaction requested=false 2024-12-03T05:00:08,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:08,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:08,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-03T05:00:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-03T05:00:08,443 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-03T05:00:08,443 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6600 sec 2024-12-03T05:00:08,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.6680 sec 2024-12-03T05:00:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:08,458 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:08,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-03T05:00:08,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032874359fb0e043508e6449cb60887c2a_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202008452/Put/seqid=0 2024-12-03T05:00:08,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741956_1132 (size=14594) 2024-12-03T05:00:08,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202068571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202068572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202068580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202068690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202068690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202068703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202068901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202068901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202068911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:00:08,941 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:08,941 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-03T05:00:08,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:00:08,947 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:08,949 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:08,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:08,959 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032874359fb0e043508e6449cb60887c2a_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032874359fb0e043508e6449cb60887c2a_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:08,960 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/e6fb72d41fae4e3e9a641c465cab50a5, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:08,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/e6fb72d41fae4e3e9a641c465cab50a5 is 175, key is test_row_0/A:col10/1733202008452/Put/seqid=0 2024-12-03T05:00:08,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741957_1133 (size=39549) 2024-12-03T05:00:08,989 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/e6fb72d41fae4e3e9a641c465cab50a5 2024-12-03T05:00:09,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/03e232ef23c14195890fbe13fa36b070 is 50, key is test_row_0/B:col10/1733202008452/Put/seqid=0 2024-12-03T05:00:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:00:09,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741958_1134 (size=12001) 2024-12-03T05:00:09,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/03e232ef23c14195890fbe13fa36b070 2024-12-03T05:00:09,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/ab052b931cab4f6fb727272d86f59335 is 50, key is test_row_0/C:col10/1733202008452/Put/seqid=0 2024-12-03T05:00:09,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T05:00:09,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:09,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:09,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:09,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:09,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:09,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741959_1135 (size=12001) 2024-12-03T05:00:09,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/ab052b931cab4f6fb727272d86f59335 2024-12-03T05:00:09,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/e6fb72d41fae4e3e9a641c465cab50a5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e6fb72d41fae4e3e9a641c465cab50a5 2024-12-03T05:00:09,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e6fb72d41fae4e3e9a641c465cab50a5, entries=200, sequenceid=55, filesize=38.6 K 2024-12-03T05:00:09,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/03e232ef23c14195890fbe13fa36b070 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/03e232ef23c14195890fbe13fa36b070 2024-12-03T05:00:09,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/03e232ef23c14195890fbe13fa36b070, entries=150, sequenceid=55, filesize=11.7 K 2024-12-03T05:00:09,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/ab052b931cab4f6fb727272d86f59335 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ab052b931cab4f6fb727272d86f59335 2024-12-03T05:00:09,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202069141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202069150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,157 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in b4ec74c11861541ec3b2d9523601012e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:09,157 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in b4ec74c11861541ec3b2d9523601012e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ab052b931cab4f6fb727272d86f59335, entries=150, sequenceid=55, filesize=11.7 K 2024-12-03T05:00:09,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b4ec74c11861541ec3b2d9523601012e in 704ms, sequenceid=55, compaction requested=true 2024-12-03T05:00:09,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:09,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:09,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:09,162 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:09,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:09,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:09,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:09,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:00:09,163 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:09,164 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:09,164 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/A is initiating minor compaction (all files) 2024-12-03T05:00:09,164 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/A in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:09,164 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d32c093578d94c24bba02c5562d91a9d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d2f9ad92af084427a6773ac067f70fea, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e6fb72d41fae4e3e9a641c465cab50a5] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=115.9 K 2024-12-03T05:00:09,165 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:09,165 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d32c093578d94c24bba02c5562d91a9d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d2f9ad92af084427a6773ac067f70fea, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e6fb72d41fae4e3e9a641c465cab50a5] 2024-12-03T05:00:09,165 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:09,165 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/B is initiating minor compaction (all files) 2024-12-03T05:00:09,165 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32c093578d94c24bba02c5562d91a9d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733202006804 2024-12-03T05:00:09,165 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/B in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:09,166 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/21206a532e384a1aabc18ba1876f6b34, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/464090caf3fe4e279c08c00826594c19, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/03e232ef23c14195890fbe13fa36b070] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=35.2 K 2024-12-03T05:00:09,166 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2f9ad92af084427a6773ac067f70fea, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733202006923 2024-12-03T05:00:09,166 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 21206a532e384a1aabc18ba1876f6b34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733202006804 2024-12-03T05:00:09,166 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6fb72d41fae4e3e9a641c465cab50a5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733202008103 2024-12-03T05:00:09,167 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 464090caf3fe4e279c08c00826594c19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733202006923 2024-12-03T05:00:09,168 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 03e232ef23c14195890fbe13fa36b070, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733202008103 2024-12-03T05:00:09,180 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#B#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:09,181 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/115ac88266914c459542892749957af0 is 50, key is test_row_0/B:col10/1733202008452/Put/seqid=0 2024-12-03T05:00:09,198 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:09,206 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412035325d913bc8343c0b190a807b2748c16_b4ec74c11861541ec3b2d9523601012e store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:09,213 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412035325d913bc8343c0b190a807b2748c16_b4ec74c11861541ec3b2d9523601012e, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:09,213 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035325d913bc8343c0b190a807b2748c16_b4ec74c11861541ec3b2d9523601012e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:09,224 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:09,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-03T05:00:09,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741960_1136 (size=12104) 2024-12-03T05:00:09,243 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/115ac88266914c459542892749957af0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/115ac88266914c459542892749957af0 2024-12-03T05:00:09,250 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/B of b4ec74c11861541ec3b2d9523601012e into 115ac88266914c459542892749957af0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:09,250 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:09,250 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/B, priority=13, startTime=1733202009162; duration=0sec 2024-12-03T05:00:09,250 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:09,250 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:B 2024-12-03T05:00:09,251 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:09,254 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:09,254 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/C is initiating minor compaction (all files) 2024-12-03T05:00:09,254 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/C in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:09,254 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/487620b102df4aa2be2b0f57011cae13, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ada920a11ae54afcb3f6fe933f9fed18, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ab052b931cab4f6fb727272d86f59335] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=35.2 K 2024-12-03T05:00:09,255 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 487620b102df4aa2be2b0f57011cae13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733202006804 2024-12-03T05:00:09,255 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ada920a11ae54afcb3f6fe933f9fed18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733202006923 2024-12-03T05:00:09,256 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ab052b931cab4f6fb727272d86f59335, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733202008103 2024-12-03T05:00:09,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202069253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:00:09,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T05:00:09,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:09,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:09,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:09,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:09,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:09,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202069255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202069257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741961_1137 (size=4469) 2024-12-03T05:00:09,284 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#C#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:09,285 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/67c30e1adbc64292a16c82626edf8e98 is 50, key is test_row_0/C:col10/1733202008452/Put/seqid=0 2024-12-03T05:00:09,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d61462327f0d40df98f362fc0459c592_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202008569/Put/seqid=0 2024-12-03T05:00:09,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741962_1138 (size=14594) 2024-12-03T05:00:09,306 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:09,317 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d61462327f0d40df98f362fc0459c592_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d61462327f0d40df98f362fc0459c592_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:09,318 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/b85920c6b7884e87b47c0a55c0f94300, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:09,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/b85920c6b7884e87b47c0a55c0f94300 is 175, key is test_row_0/A:col10/1733202008569/Put/seqid=0 2024-12-03T05:00:09,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741963_1139 (size=12104) 2024-12-03T05:00:09,343 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/67c30e1adbc64292a16c82626edf8e98 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/67c30e1adbc64292a16c82626edf8e98 2024-12-03T05:00:09,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741964_1140 (size=39549) 2024-12-03T05:00:09,350 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/C of b4ec74c11861541ec3b2d9523601012e into 67c30e1adbc64292a16c82626edf8e98(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:09,350 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:09,350 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/C, priority=13, startTime=1733202009163; duration=0sec 2024-12-03T05:00:09,350 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:09,351 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:C 2024-12-03T05:00:09,351 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/b85920c6b7884e87b47c0a55c0f94300 2024-12-03T05:00:09,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202069360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/a0d142eaa2cc426a9299cbff29acd5fa is 50, key is test_row_0/B:col10/1733202008569/Put/seqid=0 2024-12-03T05:00:09,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741965_1141 (size=12001) 2024-12-03T05:00:09,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/a0d142eaa2cc426a9299cbff29acd5fa 2024-12-03T05:00:09,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202069382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202069382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/7e78c0da0c654f0f98861aae0f6f0602 is 50, key is test_row_0/C:col10/1733202008569/Put/seqid=0 2024-12-03T05:00:09,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741966_1142 (size=12001) 2024-12-03T05:00:09,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/7e78c0da0c654f0f98861aae0f6f0602 2024-12-03T05:00:09,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/b85920c6b7884e87b47c0a55c0f94300 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b85920c6b7884e87b47c0a55c0f94300 2024-12-03T05:00:09,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T05:00:09,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:09,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:09,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:09,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:09,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b85920c6b7884e87b47c0a55c0f94300, entries=200, sequenceid=79, filesize=38.6 K 2024-12-03T05:00:09,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/a0d142eaa2cc426a9299cbff29acd5fa as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/a0d142eaa2cc426a9299cbff29acd5fa 2024-12-03T05:00:09,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/a0d142eaa2cc426a9299cbff29acd5fa, entries=150, sequenceid=79, filesize=11.7 K 2024-12-03T05:00:09,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/7e78c0da0c654f0f98861aae0f6f0602 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/7e78c0da0c654f0f98861aae0f6f0602 2024-12-03T05:00:09,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/7e78c0da0c654f0f98861aae0f6f0602, entries=150, sequenceid=79, filesize=11.7 K 2024-12-03T05:00:09,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b4ec74c11861541ec3b2d9523601012e in 231ms, sequenceid=79, compaction requested=false 2024-12-03T05:00:09,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:00:09,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T05:00:09,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:09,582 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-03T05:00:09,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:09,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dcae453076f442feb9a7ece38ea9f75d_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202009251/Put/seqid=0 2024-12-03T05:00:09,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741967_1143 (size=12154) 2024-12-03T05:00:09,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:09,675 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#A#compaction#118 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:09,676 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/eb191d313ea8488ba6ab479103b79ce9 is 175, key is test_row_0/A:col10/1733202008452/Put/seqid=0 2024-12-03T05:00:09,678 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dcae453076f442feb9a7ece38ea9f75d_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dcae453076f442feb9a7ece38ea9f75d_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:09,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1197af8f279b445daa73f46c53294344, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:09,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1197af8f279b445daa73f46c53294344 is 175, key is test_row_0/A:col10/1733202009251/Put/seqid=0 2024-12-03T05:00:09,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202069682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202069687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202069693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741968_1144 (size=31058) 2024-12-03T05:00:09,716 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/eb191d313ea8488ba6ab479103b79ce9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb191d313ea8488ba6ab479103b79ce9 2024-12-03T05:00:09,732 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/A of b4ec74c11861541ec3b2d9523601012e into eb191d313ea8488ba6ab479103b79ce9(size=30.3 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:09,732 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:09,732 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/A, priority=13, startTime=1733202009162; duration=0sec 2024-12-03T05:00:09,732 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:09,732 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:A 2024-12-03T05:00:09,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741969_1145 (size=30955) 2024-12-03T05:00:09,742 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1197af8f279b445daa73f46c53294344 2024-12-03T05:00:09,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/036b91d59f164a58b0c84e037d3ebd04 is 50, key is test_row_0/B:col10/1733202009251/Put/seqid=0 2024-12-03T05:00:09,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741970_1146 (size=12001) 2024-12-03T05:00:09,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202069801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202069801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:09,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202069811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:09,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202070020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202070021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202070021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:00:10,170 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:00:10,205 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/036b91d59f164a58b0c84e037d3ebd04 2024-12-03T05:00:10,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/6f4d19d52bc948f1aa40f3408f0773a5 is 50, key is test_row_0/C:col10/1733202009251/Put/seqid=0 2024-12-03T05:00:10,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741971_1147 (size=12001) 2024-12-03T05:00:10,245 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/6f4d19d52bc948f1aa40f3408f0773a5 2024-12-03T05:00:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1197af8f279b445daa73f46c53294344 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1197af8f279b445daa73f46c53294344 2024-12-03T05:00:10,264 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1197af8f279b445daa73f46c53294344, entries=150, sequenceid=94, filesize=30.2 K 2024-12-03T05:00:10,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/036b91d59f164a58b0c84e037d3ebd04 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/036b91d59f164a58b0c84e037d3ebd04 2024-12-03T05:00:10,273 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/036b91d59f164a58b0c84e037d3ebd04, entries=150, sequenceid=94, filesize=11.7 K 2024-12-03T05:00:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/6f4d19d52bc948f1aa40f3408f0773a5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6f4d19d52bc948f1aa40f3408f0773a5 2024-12-03T05:00:10,282 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6f4d19d52bc948f1aa40f3408f0773a5, entries=150, sequenceid=94, filesize=11.7 K 2024-12-03T05:00:10,283 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b4ec74c11861541ec3b2d9523601012e in 701ms, sequenceid=94, compaction requested=true 2024-12-03T05:00:10,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:10,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:10,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-03T05:00:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-03T05:00:10,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-03T05:00:10,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3370 sec 2024-12-03T05:00:10,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.3480 sec 2024-12-03T05:00:10,335 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:10,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-03T05:00:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:10,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202070362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202070374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202070374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bd4f8aec8aef4a8c9d5dc1bb31708d2f_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202009689/Put/seqid=0 2024-12-03T05:00:10,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741972_1148 (size=14594) 2024-12-03T05:00:10,431 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:10,437 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bd4f8aec8aef4a8c9d5dc1bb31708d2f_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bd4f8aec8aef4a8c9d5dc1bb31708d2f_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:10,439 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1a4b21138fcb4c9f93eda79ec7248619, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:10,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1a4b21138fcb4c9f93eda79ec7248619 is 175, key is test_row_0/A:col10/1733202009689/Put/seqid=0 2024-12-03T05:00:10,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741973_1149 (size=39549) 2024-12-03T05:00:10,482 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1a4b21138fcb4c9f93eda79ec7248619 2024-12-03T05:00:10,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202070481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202070498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202070498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/13efb4d1e53348d6a210402dc4502f03 is 50, key is test_row_0/B:col10/1733202009689/Put/seqid=0 2024-12-03T05:00:10,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741974_1150 (size=12001) 2024-12-03T05:00:10,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/13efb4d1e53348d6a210402dc4502f03 2024-12-03T05:00:10,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/a70e5f3c436548dfaa0ead0eabb701b8 is 50, key is test_row_0/C:col10/1733202009689/Put/seqid=0 2024-12-03T05:00:10,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741975_1151 (size=12001) 2024-12-03T05:00:10,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202070713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202070713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202070720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:10,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:10,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/a70e5f3c436548dfaa0ead0eabb701b8 2024-12-03T05:00:11,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202071032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202071033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202071034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1a4b21138fcb4c9f93eda79ec7248619 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1a4b21138fcb4c9f93eda79ec7248619 2024-12-03T05:00:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:00:11,092 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:11,092 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-03T05:00:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:00:11,098 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:11,099 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:11,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:11,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1a4b21138fcb4c9f93eda79ec7248619, entries=200, sequenceid=119, filesize=38.6 K 2024-12-03T05:00:11,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/13efb4d1e53348d6a210402dc4502f03 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/13efb4d1e53348d6a210402dc4502f03 2024-12-03T05:00:11,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/13efb4d1e53348d6a210402dc4502f03, entries=150, sequenceid=119, filesize=11.7 K 2024-12-03T05:00:11,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/a70e5f3c436548dfaa0ead0eabb701b8 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a70e5f3c436548dfaa0ead0eabb701b8 2024-12-03T05:00:11,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a70e5f3c436548dfaa0ead0eabb701b8, entries=150, sequenceid=119, filesize=11.7 K 2024-12-03T05:00:11,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b4ec74c11861541ec3b2d9523601012e in 835ms, sequenceid=119, compaction requested=true 2024-12-03T05:00:11,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:11,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:11,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:11,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:11,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:00:11,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:11,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T05:00:11,177 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:11,177 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:11,184 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:11,185 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/A is initiating minor compaction (all files) 2024-12-03T05:00:11,185 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/A in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,185 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb191d313ea8488ba6ab479103b79ce9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b85920c6b7884e87b47c0a55c0f94300, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1197af8f279b445daa73f46c53294344, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1a4b21138fcb4c9f93eda79ec7248619] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=137.8 K 2024-12-03T05:00:11,185 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,185 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb191d313ea8488ba6ab479103b79ce9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b85920c6b7884e87b47c0a55c0f94300, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1197af8f279b445daa73f46c53294344, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1a4b21138fcb4c9f93eda79ec7248619] 2024-12-03T05:00:11,188 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb191d313ea8488ba6ab479103b79ce9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733202008103 2024-12-03T05:00:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:11,190 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:11,190 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/C is initiating minor compaction (all files) 2024-12-03T05:00:11,190 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/C in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,190 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:11,190 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/67c30e1adbc64292a16c82626edf8e98, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/7e78c0da0c654f0f98861aae0f6f0602, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6f4d19d52bc948f1aa40f3408f0773a5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a70e5f3c436548dfaa0ead0eabb701b8] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=47.0 K 2024-12-03T05:00:11,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-03T05:00:11,190 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b85920c6b7884e87b47c0a55c0f94300, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733202008569 2024-12-03T05:00:11,192 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c30e1adbc64292a16c82626edf8e98, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733202008103 2024-12-03T05:00:11,199 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e78c0da0c654f0f98861aae0f6f0602, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733202008569 2024-12-03T05:00:11,199 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1197af8f279b445daa73f46c53294344, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733202009251 2024-12-03T05:00:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:00:11,202 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f4d19d52bc948f1aa40f3408f0773a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733202009251 2024-12-03T05:00:11,203 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a4b21138fcb4c9f93eda79ec7248619, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733202009665 2024-12-03T05:00:11,206 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a70e5f3c436548dfaa0ead0eabb701b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733202009665 2024-12-03T05:00:11,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203983d5b5f28fd4b19891ac81beacbae0f_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202010370/Put/seqid=0 2024-12-03T05:00:11,238 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#C#compaction#129 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:11,239 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/4c83ab82746641f78159edd42e0fd4f1 is 50, key is test_row_0/C:col10/1733202009689/Put/seqid=0 2024-12-03T05:00:11,243 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:11,257 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412034b140533a4614d72b04cf241116db1f3_b4ec74c11861541ec3b2d9523601012e store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:11,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:00:11,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:11,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,266 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412034b140533a4614d72b04cf241116db1f3_b4ec74c11861541ec3b2d9523601012e, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:11,266 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034b140533a4614d72b04cf241116db1f3_b4ec74c11861541ec3b2d9523601012e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:11,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741977_1153 (size=17184) 2024-12-03T05:00:11,307 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:11,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741976_1152 (size=12241) 2024-12-03T05:00:11,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741978_1154 (size=4469) 2024-12-03T05:00:11,332 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#A#compaction#131 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:11,333 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/94bd155acec44045b9fba60c98cc78aa is 175, key is test_row_0/A:col10/1733202009689/Put/seqid=0 2024-12-03T05:00:11,334 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203983d5b5f28fd4b19891ac81beacbae0f_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203983d5b5f28fd4b19891ac81beacbae0f_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:11,341 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/4c83ab82746641f78159edd42e0fd4f1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/4c83ab82746641f78159edd42e0fd4f1 2024-12-03T05:00:11,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202071344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,349 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/64be5146cec84b20a06e32944429a2f3, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:11,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/64be5146cec84b20a06e32944429a2f3 is 175, key is test_row_0/A:col10/1733202010370/Put/seqid=0 2024-12-03T05:00:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202071346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741979_1155 (size=31195) 2024-12-03T05:00:11,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741980_1156 (size=48289) 2024-12-03T05:00:11,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/64be5146cec84b20a06e32944429a2f3 2024-12-03T05:00:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:00:11,411 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/C of b4ec74c11861541ec3b2d9523601012e into 4c83ab82746641f78159edd42e0fd4f1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:11,411 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:11,411 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/C, priority=12, startTime=1733202011171; duration=0sec 2024-12-03T05:00:11,412 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:11,412 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:C 2024-12-03T05:00:11,412 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:11,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:00:11,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:11,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,438 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:11,438 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/B is initiating minor compaction (all files) 2024-12-03T05:00:11,438 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/B in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,439 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/115ac88266914c459542892749957af0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/a0d142eaa2cc426a9299cbff29acd5fa, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/036b91d59f164a58b0c84e037d3ebd04, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/13efb4d1e53348d6a210402dc4502f03] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=47.0 K 2024-12-03T05:00:11,439 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 115ac88266914c459542892749957af0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733202008103 2024-12-03T05:00:11,440 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a0d142eaa2cc426a9299cbff29acd5fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733202008569 2024-12-03T05:00:11,441 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 036b91d59f164a58b0c84e037d3ebd04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733202009251 2024-12-03T05:00:11,441 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 13efb4d1e53348d6a210402dc4502f03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733202009665 2024-12-03T05:00:11,444 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/94bd155acec44045b9fba60c98cc78aa as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/94bd155acec44045b9fba60c98cc78aa 2024-12-03T05:00:11,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/281c3f46c3af45ebbd9065fac9b3149b is 50, key is test_row_0/B:col10/1733202010370/Put/seqid=0 2024-12-03T05:00:11,456 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/A of b4ec74c11861541ec3b2d9523601012e into 94bd155acec44045b9fba60c98cc78aa(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:11,456 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:11,456 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/A, priority=12, startTime=1733202011171; duration=0sec 2024-12-03T05:00:11,456 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:11,456 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:A 2024-12-03T05:00:11,468 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:11,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,469 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/f1d35452d0ce403abf6d5d4ebb92dc51 is 50, key is test_row_0/B:col10/1733202009689/Put/seqid=0 2024-12-03T05:00:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202071461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202071471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741982_1158 (size=12241) 2024-12-03T05:00:11,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741981_1157 (size=12101) 2024-12-03T05:00:11,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/281c3f46c3af45ebbd9065fac9b3149b 2024-12-03T05:00:11,557 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/f1d35452d0ce403abf6d5d4ebb92dc51 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/f1d35452d0ce403abf6d5d4ebb92dc51 2024-12-03T05:00:11,567 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/B of b4ec74c11861541ec3b2d9523601012e into f1d35452d0ce403abf6d5d4ebb92dc51(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:11,567 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:11,567 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/B, priority=12, startTime=1733202011171; duration=0sec 2024-12-03T05:00:11,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/e71ca8a682074deebbd23ec36ee2fe5b is 50, key is test_row_0/C:col10/1733202010370/Put/seqid=0 2024-12-03T05:00:11,567 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:11,567 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:B 2024-12-03T05:00:11,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202071570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202071571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202071572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741983_1159 (size=12101) 2024-12-03T05:00:11,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/e71ca8a682074deebbd23ec36ee2fe5b 2024-12-03T05:00:11,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:00:11,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/64be5146cec84b20a06e32944429a2f3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/64be5146cec84b20a06e32944429a2f3 2024-12-03T05:00:11,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/64be5146cec84b20a06e32944429a2f3, entries=250, sequenceid=133, filesize=47.2 K 2024-12-03T05:00:11,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/281c3f46c3af45ebbd9065fac9b3149b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/281c3f46c3af45ebbd9065fac9b3149b 2024-12-03T05:00:11,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/281c3f46c3af45ebbd9065fac9b3149b, entries=150, sequenceid=133, filesize=11.8 K 2024-12-03T05:00:11,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/e71ca8a682074deebbd23ec36ee2fe5b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e71ca8a682074deebbd23ec36ee2fe5b 2024-12-03T05:00:11,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e71ca8a682074deebbd23ec36ee2fe5b, entries=150, sequenceid=133, filesize=11.8 K 2024-12-03T05:00:11,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b4ec74c11861541ec3b2d9523601012e in 478ms, sequenceid=133, compaction requested=false 2024-12-03T05:00:11,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:11,687 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:11,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-03T05:00:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:00:11,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035fdc38e6b7784f97a5c65a80e8f6b6de_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202011309/Put/seqid=0 2024-12-03T05:00:11,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202071749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:00:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202071756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741984_1160 (size=14794) 2024-12-03T05:00:11,786 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:11,798 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035fdc38e6b7784f97a5c65a80e8f6b6de_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035fdc38e6b7784f97a5c65a80e8f6b6de_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:11,799 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/eb04a80c8f4b449386c23ea3df163a0e, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:11,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/eb04a80c8f4b449386c23ea3df163a0e is 175, key is test_row_0/A:col10/1733202011309/Put/seqid=0 2024-12-03T05:00:11,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741985_1161 (size=39749) 2024-12-03T05:00:11,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202071861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202071871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:11,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:00:11,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:11,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:11,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202072081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:12,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:00:12,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:12,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202072087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:00:12,259 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/eb04a80c8f4b449386c23ea3df163a0e 2024-12-03T05:00:12,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:00:12,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:12,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/bcb743ac1d8b482c90e1f8ae937b9448 is 50, key is test_row_0/B:col10/1733202011309/Put/seqid=0 2024-12-03T05:00:12,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741986_1162 (size=12151) 2024-12-03T05:00:12,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/bcb743ac1d8b482c90e1f8ae937b9448 2024-12-03T05:00:12,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/6c3f36ec4e9a40a4ab245c6a63c2931d is 50, key is test_row_0/C:col10/1733202011309/Put/seqid=0 2024-12-03T05:00:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202072391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:12,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:12,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:00:12,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:12,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202072411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741987_1163 (size=12151) 2024-12-03T05:00:12,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/6c3f36ec4e9a40a4ab245c6a63c2931d 2024-12-03T05:00:12,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/eb04a80c8f4b449386c23ea3df163a0e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb04a80c8f4b449386c23ea3df163a0e 2024-12-03T05:00:12,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb04a80c8f4b449386c23ea3df163a0e, entries=200, sequenceid=160, filesize=38.8 K 2024-12-03T05:00:12,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/bcb743ac1d8b482c90e1f8ae937b9448 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bcb743ac1d8b482c90e1f8ae937b9448 2024-12-03T05:00:12,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bcb743ac1d8b482c90e1f8ae937b9448, entries=150, sequenceid=160, filesize=11.9 K 2024-12-03T05:00:12,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/6c3f36ec4e9a40a4ab245c6a63c2931d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6c3f36ec4e9a40a4ab245c6a63c2931d 2024-12-03T05:00:12,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:00:12,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:12,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:12,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202072591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:12,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202072591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:12,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:12,597 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_1' in b4ec74c11861541ec3b2d9523601012e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,597 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in b4ec74c11861541ec3b2d9523601012e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202072591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:12,602 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in b4ec74c11861541ec3b2d9523601012e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:12,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6c3f36ec4e9a40a4ab245c6a63c2931d, entries=150, sequenceid=160, filesize=11.9 K 2024-12-03T05:00:12,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b4ec74c11861541ec3b2d9523601012e in 934ms, sequenceid=160, compaction requested=true 2024-12-03T05:00:12,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:12,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:12,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:12,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:12,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:12,622 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:12,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:12,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:00:12,622 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:12,639 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:12,639 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/A is initiating minor compaction (all files) 2024-12-03T05:00:12,639 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/A in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,639 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/94bd155acec44045b9fba60c98cc78aa, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/64be5146cec84b20a06e32944429a2f3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb04a80c8f4b449386c23ea3df163a0e] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=116.4 K 2024-12-03T05:00:12,639 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,640 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/94bd155acec44045b9fba60c98cc78aa, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/64be5146cec84b20a06e32944429a2f3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb04a80c8f4b449386c23ea3df163a0e] 2024-12-03T05:00:12,642 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:12,642 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/B is initiating minor compaction (all files) 2024-12-03T05:00:12,642 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/B in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,642 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/f1d35452d0ce403abf6d5d4ebb92dc51, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/281c3f46c3af45ebbd9065fac9b3149b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bcb743ac1d8b482c90e1f8ae937b9448] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=35.6 K 2024-12-03T05:00:12,649 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94bd155acec44045b9fba60c98cc78aa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733202009665 2024-12-03T05:00:12,657 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting f1d35452d0ce403abf6d5d4ebb92dc51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733202009665 2024-12-03T05:00:12,658 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64be5146cec84b20a06e32944429a2f3, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733202010358 2024-12-03T05:00:12,666 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 281c3f46c3af45ebbd9065fac9b3149b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733202010359 2024-12-03T05:00:12,666 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb04a80c8f4b449386c23ea3df163a0e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733202011309 2024-12-03T05:00:12,671 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting bcb743ac1d8b482c90e1f8ae937b9448, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733202011309 2024-12-03T05:00:12,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:00:12,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,757 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-03T05:00:12,759 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:12,760 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/1ae3d7a0a6f54908af1bc73d64db9671 is 50, key is test_row_0/B:col10/1733202011309/Put/seqid=0 2024-12-03T05:00:12,766 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:12,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741988_1164 (size=12493) 2024-12-03T05:00:12,838 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203a0fc2468244f466f8ac293d109f289bf_b4ec74c11861541ec3b2d9523601012e store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:12,840 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203a0fc2468244f466f8ac293d109f289bf_b4ec74c11861541ec3b2d9523601012e, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:12,841 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a0fc2468244f466f8ac293d109f289bf_b4ec74c11861541ec3b2d9523601012e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:12,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e1d6850ee5a4421593d2ace9feaf1843_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202011735/Put/seqid=0 2024-12-03T05:00:12,865 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/1ae3d7a0a6f54908af1bc73d64db9671 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/1ae3d7a0a6f54908af1bc73d64db9671 2024-12-03T05:00:12,872 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/B of b4ec74c11861541ec3b2d9523601012e into 1ae3d7a0a6f54908af1bc73d64db9671(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:12,872 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:12,872 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/B, priority=13, startTime=1733202012622; duration=0sec 2024-12-03T05:00:12,872 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:12,872 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:B 2024-12-03T05:00:12,873 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:12,885 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:12,886 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/C is initiating minor compaction (all files) 2024-12-03T05:00:12,886 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/C in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:12,886 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/4c83ab82746641f78159edd42e0fd4f1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e71ca8a682074deebbd23ec36ee2fe5b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6c3f36ec4e9a40a4ab245c6a63c2931d] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=35.6 K 2024-12-03T05:00:12,887 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c83ab82746641f78159edd42e0fd4f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733202009665 2024-12-03T05:00:12,889 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting e71ca8a682074deebbd23ec36ee2fe5b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733202010359 2024-12-03T05:00:12,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741989_1165 (size=4469) 2024-12-03T05:00:12,897 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c3f36ec4e9a40a4ab245c6a63c2931d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733202011309 2024-12-03T05:00:12,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741990_1166 (size=12304) 2024-12-03T05:00:12,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:12,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:12,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:12,931 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#C#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:12,932 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/ecbf73f84e1e4a029bdb63e9af455736 is 50, key is test_row_0/C:col10/1733202011309/Put/seqid=0 2024-12-03T05:00:12,935 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e1d6850ee5a4421593d2ace9feaf1843_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e1d6850ee5a4421593d2ace9feaf1843_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:12,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/6654c78bda6d4f9d8f80fc24a776daae, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:12,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/6654c78bda6d4f9d8f80fc24a776daae is 175, key is test_row_0/A:col10/1733202011735/Put/seqid=0 2024-12-03T05:00:12,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741991_1167 (size=12493) 2024-12-03T05:00:13,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741992_1168 (size=31105) 2024-12-03T05:00:13,021 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/6654c78bda6d4f9d8f80fc24a776daae 2024-12-03T05:00:13,029 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/ecbf73f84e1e4a029bdb63e9af455736 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ecbf73f84e1e4a029bdb63e9af455736 2024-12-03T05:00:13,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202073042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:13,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202073045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:13,060 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/C of b4ec74c11861541ec3b2d9523601012e into ecbf73f84e1e4a029bdb63e9af455736(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:13,060 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:13,060 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/C, priority=13, startTime=1733202012622; duration=0sec 2024-12-03T05:00:13,060 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:13,060 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:C 2024-12-03T05:00:13,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/312f9eb69c34483782f306d12228c93f is 50, key is test_row_0/B:col10/1733202011735/Put/seqid=0 2024-12-03T05:00:13,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741993_1169 (size=12151) 2024-12-03T05:00:13,153 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/312f9eb69c34483782f306d12228c93f 2024-12-03T05:00:13,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:13,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202073161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:13,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:13,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202073161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:13,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/8774b27be3eb47d8b6f408e664a26ad0 is 50, key is test_row_0/C:col10/1733202011735/Put/seqid=0 2024-12-03T05:00:13,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741994_1170 (size=12151) 2024-12-03T05:00:13,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:00:13,254 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/8774b27be3eb47d8b6f408e664a26ad0 2024-12-03T05:00:13,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/6654c78bda6d4f9d8f80fc24a776daae as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/6654c78bda6d4f9d8f80fc24a776daae 2024-12-03T05:00:13,284 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/6654c78bda6d4f9d8f80fc24a776daae, entries=150, sequenceid=172, filesize=30.4 K 2024-12-03T05:00:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/312f9eb69c34483782f306d12228c93f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/312f9eb69c34483782f306d12228c93f 2024-12-03T05:00:13,304 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#A#compaction#139 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:13,305 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/0dd71e170ca04a17b714de7dadce6af5 is 175, key is test_row_0/A:col10/1733202011309/Put/seqid=0 2024-12-03T05:00:13,313 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/312f9eb69c34483782f306d12228c93f, entries=150, sequenceid=172, filesize=11.9 K 2024-12-03T05:00:13,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/8774b27be3eb47d8b6f408e664a26ad0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8774b27be3eb47d8b6f408e664a26ad0 2024-12-03T05:00:13,336 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8774b27be3eb47d8b6f408e664a26ad0, entries=150, sequenceid=172, filesize=11.9 K 2024-12-03T05:00:13,338 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b4ec74c11861541ec3b2d9523601012e in 581ms, sequenceid=172, compaction requested=false 2024-12-03T05:00:13,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:13,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:13,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-03T05:00:13,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741995_1171 (size=31447) 2024-12-03T05:00:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-03T05:00:13,351 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/0dd71e170ca04a17b714de7dadce6af5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/0dd71e170ca04a17b714de7dadce6af5 2024-12-03T05:00:13,358 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/A of b4ec74c11861541ec3b2d9523601012e into 0dd71e170ca04a17b714de7dadce6af5(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:13,358 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:13,358 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/A, priority=13, startTime=1733202012621; duration=0sec 2024-12-03T05:00:13,358 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:13,358 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:A 2024-12-03T05:00:13,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.2640 sec 2024-12-03T05:00:13,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-03T05:00:13,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2510 sec 2024-12-03T05:00:13,391 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:13,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:00:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:13,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203075c58c518234d01914e266da6ab2252_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202013386/Put/seqid=0 2024-12-03T05:00:13,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202073451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:13,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202073453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:13,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:13,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741996_1172 (size=12304) 2024-12-03T05:00:13,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202073563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:13,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:13,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202073568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:13,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202073773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:13,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202073780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:13,860 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:13,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203075c58c518234d01914e266da6ab2252_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203075c58c518234d01914e266da6ab2252_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:13,871 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1392dcdae5da4ebeaadc3feddf7c54f2, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:13,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1392dcdae5da4ebeaadc3feddf7c54f2 is 175, key is test_row_0/A:col10/1733202013386/Put/seqid=0 2024-12-03T05:00:13,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741997_1173 (size=31105) 2024-12-03T05:00:13,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1392dcdae5da4ebeaadc3feddf7c54f2 2024-12-03T05:00:13,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/27730a0a7ea34083be502641c4cd0a2d is 50, key is test_row_0/B:col10/1733202013386/Put/seqid=0 2024-12-03T05:00:13,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741998_1174 (size=12151) 2024-12-03T05:00:14,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202074098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:14,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202074098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:14,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/27730a0a7ea34083be502641c4cd0a2d 2024-12-03T05:00:14,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/ddf4df2dd1d541ed827029b6760ff07b is 50, key is test_row_0/C:col10/1733202013386/Put/seqid=0 2024-12-03T05:00:14,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073741999_1175 (size=12151) 2024-12-03T05:00:14,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202074611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:14,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202074611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:14,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202074613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:14,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202074617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:14,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202074617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:14,618 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in b4ec74c11861541ec3b2d9523601012e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:14,618 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in b4ec74c11861541ec3b2d9523601012e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:14,618 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_1' in b4ec74c11861541ec3b2d9523601012e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:14,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/ddf4df2dd1d541ed827029b6760ff07b 2024-12-03T05:00:14,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1392dcdae5da4ebeaadc3feddf7c54f2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1392dcdae5da4ebeaadc3feddf7c54f2 2024-12-03T05:00:14,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1392dcdae5da4ebeaadc3feddf7c54f2, entries=150, sequenceid=200, filesize=30.4 K 2024-12-03T05:00:14,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/27730a0a7ea34083be502641c4cd0a2d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/27730a0a7ea34083be502641c4cd0a2d 2024-12-03T05:00:14,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/27730a0a7ea34083be502641c4cd0a2d, entries=150, sequenceid=200, filesize=11.9 K 2024-12-03T05:00:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/ddf4df2dd1d541ed827029b6760ff07b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ddf4df2dd1d541ed827029b6760ff07b 2024-12-03T05:00:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ddf4df2dd1d541ed827029b6760ff07b, entries=150, sequenceid=200, filesize=11.9 K 2024-12-03T05:00:14,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for b4ec74c11861541ec3b2d9523601012e in 1545ms, sequenceid=200, compaction requested=true 2024-12-03T05:00:14,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:14,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:14,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:14,937 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:14,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:14,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:14,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:14,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:14,937 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:14,940 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:14,940 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/A is initiating minor compaction (all files) 2024-12-03T05:00:14,940 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/A in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:14,941 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/0dd71e170ca04a17b714de7dadce6af5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/6654c78bda6d4f9d8f80fc24a776daae, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1392dcdae5da4ebeaadc3feddf7c54f2] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=91.5 K 2024-12-03T05:00:14,941 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:14,941 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/0dd71e170ca04a17b714de7dadce6af5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/6654c78bda6d4f9d8f80fc24a776daae, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1392dcdae5da4ebeaadc3feddf7c54f2] 2024-12-03T05:00:14,942 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dd71e170ca04a17b714de7dadce6af5, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733202011309 2024-12-03T05:00:14,943 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6654c78bda6d4f9d8f80fc24a776daae, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733202011722 2024-12-03T05:00:14,943 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1392dcdae5da4ebeaadc3feddf7c54f2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733202013015 2024-12-03T05:00:14,949 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:14,949 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/B is initiating minor compaction (all files) 2024-12-03T05:00:14,949 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/B in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:14,949 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/1ae3d7a0a6f54908af1bc73d64db9671, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/312f9eb69c34483782f306d12228c93f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/27730a0a7ea34083be502641c4cd0a2d] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=35.9 K 2024-12-03T05:00:14,954 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ae3d7a0a6f54908af1bc73d64db9671, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733202011309 2024-12-03T05:00:14,954 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 312f9eb69c34483782f306d12228c93f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733202011722 2024-12-03T05:00:14,955 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 27730a0a7ea34083be502641c4cd0a2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733202013015 2024-12-03T05:00:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,967 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,981 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#B#compaction#148 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,984 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/dd028eb28dce43efbfdcdc4d97c4dd03 is 50, key is test_row_0/B:col10/1733202013386/Put/seqid=0 2024-12-03T05:00:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,985 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412030478b929ce92462dba0553e77d29ca91_b4ec74c11861541ec3b2d9523601012e store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,992 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412030478b929ce92462dba0553e77d29ca91_b4ec74c11861541ec3b2d9523601012e, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:14,992 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030478b929ce92462dba0553e77d29ca91_b4ec74c11861541ec3b2d9523601012e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742001_1177 (size=4469) 2024-12-03T05:00:15,058 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#A#compaction#147 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,059 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/01d145b0557b4c9db30d78d298cfd171 is 175, key is test_row_0/A:col10/1733202013386/Put/seqid=0 2024-12-03T05:00:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742000_1176 (size=12595) 2024-12-03T05:00:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742002_1178 (size=31549) 2024-12-03T05:00:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,120 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/01d145b0557b4c9db30d78d298cfd171 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/01d145b0557b4c9db30d78d298cfd171 2024-12-03T05:00:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,161 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/A of b4ec74c11861541ec3b2d9523601012e into 01d145b0557b4c9db30d78d298cfd171(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:15,161 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:15,161 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/A, priority=13, startTime=1733202014937; duration=0sec 2024-12-03T05:00:15,161 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:15,161 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:A 2024-12-03T05:00:15,162 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,173 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:15,173 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/C is initiating minor compaction (all files) 2024-12-03T05:00:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,173 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/C in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:15,174 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ecbf73f84e1e4a029bdb63e9af455736, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8774b27be3eb47d8b6f408e664a26ad0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ddf4df2dd1d541ed827029b6760ff07b] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=35.9 K 2024-12-03T05:00:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,177 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecbf73f84e1e4a029bdb63e9af455736, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733202011309 2024-12-03T05:00:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,184 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8774b27be3eb47d8b6f408e664a26ad0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733202011722 2024-12-03T05:00:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,185 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddf4df2dd1d541ed827029b6760ff07b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733202013015 2024-12-03T05:00:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,228 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#C#compaction#149 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,229 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/8f62f5c9cb6f4e5eb3964e7b99400688 is 50, key is test_row_0/C:col10/1733202013386/Put/seqid=0 2024-12-03T05:00:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742003_1179 (size=12595) 2024-12-03T05:00:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:00:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,274 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:15,275 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/8f62f5c9cb6f4e5eb3964e7b99400688 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8f62f5c9cb6f4e5eb3964e7b99400688 2024-12-03T05:00:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,275 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-03T05:00:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,278 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:00:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,279 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:15,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,286 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/C of b4ec74c11861541ec3b2d9523601012e into 8f62f5c9cb6f4e5eb3964e7b99400688(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,286 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:15,286 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/C, priority=13, startTime=1733202014938; duration=0sec 2024-12-03T05:00:15,286 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:15,286 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:C 2024-12-03T05:00:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:00:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:00:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:15,434 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:00:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120378b9c6242de344c59d794a2c37573e4a_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_1/A:col10/1733202013436/Put/seqid=0 2024-12-03T05:00:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,482 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/dd028eb28dce43efbfdcdc4d97c4dd03 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/dd028eb28dce43efbfdcdc4d97c4dd03 2024-12-03T05:00:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,490 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/B of b4ec74c11861541ec3b2d9523601012e into dd028eb28dce43efbfdcdc4d97c4dd03(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,490 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:15,490 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/B, priority=13, startTime=1733202014937; duration=0sec 2024-12-03T05:00:15,490 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:15,490 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:B 2024-12-03T05:00:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742004_1180 (size=9814) 2024-12-03T05:00:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,514 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120378b9c6242de344c59d794a2c37573e4a_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120378b9c6242de344c59d794a2c37573e4a_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/5eec0115e66443a0871c346c09d77974, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/5eec0115e66443a0871c346c09d77974 is 175, key is test_row_1/A:col10/1733202013436/Put/seqid=0 2024-12-03T05:00:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742005_1181 (size=22461) 2024-12-03T05:00:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:00:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202075820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:15,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202075828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:00:15,945 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/5eec0115e66443a0871c346c09d77974 2024-12-03T05:00:15,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202075942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:15,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202075944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:15,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/cfe56801b5f149798a86a1e0ca06fbbf is 50, key is test_row_1/B:col10/1733202013436/Put/seqid=0 2024-12-03T05:00:16,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742006_1182 (size=9757) 2024-12-03T05:00:16,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202076161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:16,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202076161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:16,415 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/cfe56801b5f149798a86a1e0ca06fbbf 2024-12-03T05:00:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:00:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/bad8198fcb1045c7aa42f9bda5163be4 is 50, key is test_row_1/C:col10/1733202013436/Put/seqid=0 2024-12-03T05:00:16,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742007_1183 (size=9757) 2024-12-03T05:00:16,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202076470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:16,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:16,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:16,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:16,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202076480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:16,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T05:00:16,689 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T05:00:16,866 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/bad8198fcb1045c7aa42f9bda5163be4 2024-12-03T05:00:16,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/5eec0115e66443a0871c346c09d77974 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/5eec0115e66443a0871c346c09d77974 2024-12-03T05:00:16,887 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/5eec0115e66443a0871c346c09d77974, entries=100, sequenceid=213, filesize=21.9 K 2024-12-03T05:00:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/cfe56801b5f149798a86a1e0ca06fbbf as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cfe56801b5f149798a86a1e0ca06fbbf 2024-12-03T05:00:16,895 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cfe56801b5f149798a86a1e0ca06fbbf, entries=100, sequenceid=213, filesize=9.5 K 2024-12-03T05:00:16,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/bad8198fcb1045c7aa42f9bda5163be4 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/bad8198fcb1045c7aa42f9bda5163be4 2024-12-03T05:00:16,906 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/bad8198fcb1045c7aa42f9bda5163be4, entries=100, sequenceid=213, filesize=9.5 K 2024-12-03T05:00:16,907 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for b4ec74c11861541ec3b2d9523601012e in 1473ms, sequenceid=213, compaction requested=false 2024-12-03T05:00:16,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:16,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:16,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-03T05:00:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-03T05:00:16,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-03T05:00:16,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6300 sec 2024-12-03T05:00:16,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6430 sec 2024-12-03T05:00:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:16,986 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:16,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-03T05:00:17,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202077012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:17,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202077013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:17,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120335249c7270d244d28908e0bbf3b80583_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202016984/Put/seqid=0 2024-12-03T05:00:17,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742008_1184 (size=12304) 2024-12-03T05:00:17,040 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:17,047 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120335249c7270d244d28908e0bbf3b80583_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120335249c7270d244d28908e0bbf3b80583_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:17,049 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/32ef627a49c943e69363f7485bc961e8, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:17,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/32ef627a49c943e69363f7485bc961e8 is 175, key is test_row_0/A:col10/1733202016984/Put/seqid=0 2024-12-03T05:00:17,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742009_1185 (size=31105) 2024-12-03T05:00:17,075 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/32ef627a49c943e69363f7485bc961e8 2024-12-03T05:00:17,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/525729aa63ab427394fa923d7289268e is 50, key is test_row_0/B:col10/1733202016984/Put/seqid=0 2024-12-03T05:00:17,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202077121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:17,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202077121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:17,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742010_1186 (size=12151) 2024-12-03T05:00:17,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/525729aa63ab427394fa923d7289268e 2024-12-03T05:00:17,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/93d45f622da44eeea159a7eebc1c5b9a is 50, key is test_row_0/C:col10/1733202016984/Put/seqid=0 2024-12-03T05:00:17,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742011_1187 (size=12151) 2024-12-03T05:00:17,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202077331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:17,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202077333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:17,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:00:17,430 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:17,431 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-03T05:00:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T05:00:17,435 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:17,436 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:17,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T05:00:17,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/93d45f622da44eeea159a7eebc1c5b9a 2024-12-03T05:00:17,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/32ef627a49c943e69363f7485bc961e8 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/32ef627a49c943e69363f7485bc961e8 2024-12-03T05:00:17,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T05:00:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:17,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:17,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/32ef627a49c943e69363f7485bc961e8, entries=150, sequenceid=241, filesize=30.4 K 2024-12-03T05:00:17,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/525729aa63ab427394fa923d7289268e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/525729aa63ab427394fa923d7289268e 2024-12-03T05:00:17,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/525729aa63ab427394fa923d7289268e, entries=150, sequenceid=241, filesize=11.9 K 2024-12-03T05:00:17,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/93d45f622da44eeea159a7eebc1c5b9a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/93d45f622da44eeea159a7eebc1c5b9a 2024-12-03T05:00:17,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202077641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:17,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202077641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:17,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/93d45f622da44eeea159a7eebc1c5b9a, entries=150, sequenceid=241, filesize=11.9 K 2024-12-03T05:00:17,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for b4ec74c11861541ec3b2d9523601012e in 660ms, sequenceid=241, compaction requested=true 2024-12-03T05:00:17,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:17,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:17,647 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:17,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:17,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:17,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:17,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:17,647 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:17,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:17,648 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:17,648 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/A is initiating minor compaction (all files) 2024-12-03T05:00:17,648 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/A in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:17,649 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/01d145b0557b4c9db30d78d298cfd171, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/5eec0115e66443a0871c346c09d77974, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/32ef627a49c943e69363f7485bc961e8] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=83.1 K 2024-12-03T05:00:17,649 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:17,649 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/01d145b0557b4c9db30d78d298cfd171, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/5eec0115e66443a0871c346c09d77974, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/32ef627a49c943e69363f7485bc961e8] 2024-12-03T05:00:17,649 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:17,649 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/B is initiating minor compaction (all files) 2024-12-03T05:00:17,649 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/B in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:17,649 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/dd028eb28dce43efbfdcdc4d97c4dd03, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cfe56801b5f149798a86a1e0ca06fbbf, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/525729aa63ab427394fa923d7289268e] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=33.7 K 2024-12-03T05:00:17,650 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01d145b0557b4c9db30d78d298cfd171, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733202013015 2024-12-03T05:00:17,650 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting dd028eb28dce43efbfdcdc4d97c4dd03, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733202013015 2024-12-03T05:00:17,650 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eec0115e66443a0871c346c09d77974, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733202013436 2024-12-03T05:00:17,651 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting cfe56801b5f149798a86a1e0ca06fbbf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733202013436 2024-12-03T05:00:17,651 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32ef627a49c943e69363f7485bc961e8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733202015770 2024-12-03T05:00:17,652 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 525729aa63ab427394fa923d7289268e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733202015770 2024-12-03T05:00:17,662 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:17,667 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:17,668 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/66634bc3972b4304b4f7b87f02832650 is 50, key is test_row_0/B:col10/1733202016984/Put/seqid=0 2024-12-03T05:00:17,677 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120339a2d5de5e4c4552ab94d527aa4b2135_b4ec74c11861541ec3b2d9523601012e store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:17,681 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120339a2d5de5e4c4552ab94d527aa4b2135_b4ec74c11861541ec3b2d9523601012e, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:17,681 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120339a2d5de5e4c4552ab94d527aa4b2135_b4ec74c11861541ec3b2d9523601012e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:17,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742012_1188 (size=12697) 2024-12-03T05:00:17,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742013_1189 (size=4469) 2024-12-03T05:00:17,743 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#A#compaction#156 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:17,744 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/a9749b8aafc849249d6807726fa27ec2 is 175, key is test_row_0/A:col10/1733202016984/Put/seqid=0 2024-12-03T05:00:17,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T05:00:17,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:17,746 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-03T05:00:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T05:00:17,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742014_1190 (size=31651) 2024-12-03T05:00:17,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c948c76dc7e14d48b4100601d068b7d6_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202017008/Put/seqid=0 2024-12-03T05:00:17,808 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/a9749b8aafc849249d6807726fa27ec2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a9749b8aafc849249d6807726fa27ec2 2024-12-03T05:00:17,822 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/A of b4ec74c11861541ec3b2d9523601012e into a9749b8aafc849249d6807726fa27ec2(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:17,822 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:17,822 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/A, priority=13, startTime=1733202017647; duration=0sec 2024-12-03T05:00:17,822 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:17,822 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:A 2024-12-03T05:00:17,822 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:17,825 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:17,825 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/C is initiating minor compaction (all files) 2024-12-03T05:00:17,825 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/C in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:17,825 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8f62f5c9cb6f4e5eb3964e7b99400688, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/bad8198fcb1045c7aa42f9bda5163be4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/93d45f622da44eeea159a7eebc1c5b9a] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=33.7 K 2024-12-03T05:00:17,827 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f62f5c9cb6f4e5eb3964e7b99400688, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733202013015 2024-12-03T05:00:17,827 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad8198fcb1045c7aa42f9bda5163be4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733202013436 2024-12-03T05:00:17,829 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d45f622da44eeea159a7eebc1c5b9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733202015770 2024-12-03T05:00:17,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742015_1191 (size=12304) 2024-12-03T05:00:17,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:17,848 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c948c76dc7e14d48b4100601d068b7d6_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c948c76dc7e14d48b4100601d068b7d6_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:17,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/fd888b90e23e47cb81cff0b7164b6aa0, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:17,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/fd888b90e23e47cb81cff0b7164b6aa0 is 175, key is test_row_0/A:col10/1733202017008/Put/seqid=0 2024-12-03T05:00:17,859 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#C#compaction#159 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:17,860 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/a51c0745e5a144fe9ec562bfa6663c45 is 50, key is test_row_0/C:col10/1733202016984/Put/seqid=0 2024-12-03T05:00:17,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742017_1193 (size=12697) 2024-12-03T05:00:17,902 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/a51c0745e5a144fe9ec562bfa6663c45 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a51c0745e5a144fe9ec562bfa6663c45 2024-12-03T05:00:17,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742016_1192 (size=31105) 2024-12-03T05:00:17,914 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/C of b4ec74c11861541ec3b2d9523601012e into a51c0745e5a144fe9ec562bfa6663c45(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:17,914 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:17,914 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/C, priority=13, startTime=1733202017647; duration=0sec 2024-12-03T05:00:17,914 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:17,914 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:C 2024-12-03T05:00:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T05:00:18,144 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/66634bc3972b4304b4f7b87f02832650 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/66634bc3972b4304b4f7b87f02832650 2024-12-03T05:00:18,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:18,164 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/B of b4ec74c11861541ec3b2d9523601012e into 66634bc3972b4304b4f7b87f02832650(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:18,164 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:18,164 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/B, priority=13, startTime=1733202017647; duration=0sec 2024-12-03T05:00:18,165 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:18,165 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:B 2024-12-03T05:00:18,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202078228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202078228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,307 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/fd888b90e23e47cb81cff0b7164b6aa0 2024-12-03T05:00:18,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/cb4c6723be8c4bd5a16d24eea2f62a31 is 50, key is test_row_0/B:col10/1733202017008/Put/seqid=0 2024-12-03T05:00:18,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202078341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202078342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742018_1194 (size=12151) 2024-12-03T05:00:18,369 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/cb4c6723be8c4bd5a16d24eea2f62a31 2024-12-03T05:00:18,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/864b9fc3e77c4b4e95d29ef538a3b164 is 50, key is test_row_0/C:col10/1733202017008/Put/seqid=0 2024-12-03T05:00:18,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742019_1195 (size=12151) 2024-12-03T05:00:18,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202078551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202078551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T05:00:18,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57422 deadline: 1733202078630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,633 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_1' in b4ec74c11861541ec3b2d9523601012e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57400 deadline: 1733202078640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,644 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in b4ec74c11861541ec3b2d9523601012e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57440 deadline: 1733202078663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,665 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in b4ec74c11861541ec3b2d9523601012e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8303 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,804 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/864b9fc3e77c4b4e95d29ef538a3b164 2024-12-03T05:00:18,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/fd888b90e23e47cb81cff0b7164b6aa0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/fd888b90e23e47cb81cff0b7164b6aa0 2024-12-03T05:00:18,817 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/fd888b90e23e47cb81cff0b7164b6aa0, entries=150, sequenceid=250, filesize=30.4 K 2024-12-03T05:00:18,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/cb4c6723be8c4bd5a16d24eea2f62a31 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cb4c6723be8c4bd5a16d24eea2f62a31 2024-12-03T05:00:18,823 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cb4c6723be8c4bd5a16d24eea2f62a31, entries=150, sequenceid=250, filesize=11.9 K 2024-12-03T05:00:18,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/864b9fc3e77c4b4e95d29ef538a3b164 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/864b9fc3e77c4b4e95d29ef538a3b164 2024-12-03T05:00:18,831 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/864b9fc3e77c4b4e95d29ef538a3b164, entries=150, sequenceid=250, filesize=11.9 K 2024-12-03T05:00:18,832 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for b4ec74c11861541ec3b2d9523601012e in 1086ms, sequenceid=250, compaction requested=false 2024-12-03T05:00:18,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:18,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:18,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-03T05:00:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-03T05:00:18,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-03T05:00:18,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-12-03T05:00:18,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4090 sec 2024-12-03T05:00:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:18,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:18,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-12-03T05:00:18,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a8e7913fdfd94126b4dd4917206d1588_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202018861/Put/seqid=0 2024-12-03T05:00:18,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202078878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202078878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742020_1196 (size=14994) 2024-12-03T05:00:18,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202078990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202078991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:19,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202079200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:19,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202079200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:19,298 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:19,305 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a8e7913fdfd94126b4dd4917206d1588_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a8e7913fdfd94126b4dd4917206d1588_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:19,307 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1ff21c47d1dd41009f899185a3301eb1, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:19,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1ff21c47d1dd41009f899185a3301eb1 is 175, key is test_row_0/A:col10/1733202018861/Put/seqid=0 2024-12-03T05:00:19,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742021_1197 (size=39949) 2024-12-03T05:00:19,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:19,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202079511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202079511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T05:00:19,581 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:19,582 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-03T05:00:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:00:19,585 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:19,586 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:19,586 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:00:19,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:00:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:19,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:19,754 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1ff21c47d1dd41009f899185a3301eb1 2024-12-03T05:00:19,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/b5149458c7aa49cc9c670d3dbe984a10 is 50, key is test_row_0/B:col10/1733202018861/Put/seqid=0 2024-12-03T05:00:19,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742022_1198 (size=12301) 2024-12-03T05:00:19,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/b5149458c7aa49cc9c670d3dbe984a10 2024-12-03T05:00:19,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/d4ea5719deb84b51905b509c32ed7c53 is 50, key is test_row_0/C:col10/1733202018861/Put/seqid=0 2024-12-03T05:00:19,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742023_1199 (size=12301) 2024-12-03T05:00:19,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/d4ea5719deb84b51905b509c32ed7c53 2024-12-03T05:00:19,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:00:19,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:19,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:19,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:19,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:19,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:00:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:19,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/1ff21c47d1dd41009f899185a3301eb1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1ff21c47d1dd41009f899185a3301eb1 2024-12-03T05:00:19,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1ff21c47d1dd41009f899185a3301eb1, entries=200, sequenceid=281, filesize=39.0 K 2024-12-03T05:00:19,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/b5149458c7aa49cc9c670d3dbe984a10 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/b5149458c7aa49cc9c670d3dbe984a10 2024-12-03T05:00:20,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/b5149458c7aa49cc9c670d3dbe984a10, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T05:00:20,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/d4ea5719deb84b51905b509c32ed7c53 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/d4ea5719deb84b51905b509c32ed7c53 2024-12-03T05:00:20,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/d4ea5719deb84b51905b509c32ed7c53, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T05:00:20,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for b4ec74c11861541ec3b2d9523601012e in 1161ms, sequenceid=281, compaction requested=true 2024-12-03T05:00:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:20,025 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:20,025 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:20,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:20,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:20,027 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:20,027 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/A is initiating minor compaction (all files) 2024-12-03T05:00:20,027 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/A in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,027 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a9749b8aafc849249d6807726fa27ec2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/fd888b90e23e47cb81cff0b7164b6aa0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1ff21c47d1dd41009f899185a3301eb1] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=100.3 K 2024-12-03T05:00:20,027 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,027 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a9749b8aafc849249d6807726fa27ec2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/fd888b90e23e47cb81cff0b7164b6aa0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1ff21c47d1dd41009f899185a3301eb1] 2024-12-03T05:00:20,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:20,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:20,028 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:20,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:00:20,029 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9749b8aafc849249d6807726fa27ec2, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733202015770 2024-12-03T05:00:20,030 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:20,030 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd888b90e23e47cb81cff0b7164b6aa0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733202017003 2024-12-03T05:00:20,030 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/B is initiating minor compaction (all files) 2024-12-03T05:00:20,030 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/B in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,030 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/66634bc3972b4304b4f7b87f02832650, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cb4c6723be8c4bd5a16d24eea2f62a31, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/b5149458c7aa49cc9c670d3dbe984a10] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=36.3 K 2024-12-03T05:00:20,031 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ff21c47d1dd41009f899185a3301eb1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733202018192 2024-12-03T05:00:20,032 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 66634bc3972b4304b4f7b87f02832650, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733202015770 2024-12-03T05:00:20,033 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting cb4c6723be8c4bd5a16d24eea2f62a31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733202017003 2024-12-03T05:00:20,042 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting b5149458c7aa49cc9c670d3dbe984a10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733202018192 2024-12-03T05:00:20,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cceebd1e34e5483e8c21337cfb7fd7b8_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202020026/Put/seqid=0 2024-12-03T05:00:20,055 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#B#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:20,056 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/167b25140d85438a987f2e0cdde6804e is 50, key is test_row_0/B:col10/1733202018861/Put/seqid=0 2024-12-03T05:00:20,058 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:20,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742024_1200 (size=12454) 2024-12-03T05:00:20,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:00:20,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:20,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,086 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203c3d47ba7d80c460dae3ef35ca56395d9_b4ec74c11861541ec3b2d9523601012e store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:20,088 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203c3d47ba7d80c460dae3ef35ca56395d9_b4ec74c11861541ec3b2d9523601012e, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:20,088 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c3d47ba7d80c460dae3ef35ca56395d9_b4ec74c11861541ec3b2d9523601012e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:20,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742025_1201 (size=12949) 2024-12-03T05:00:20,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742026_1202 (size=4469) 2024-12-03T05:00:20,123 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#A#compaction#167 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:20,124 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/ee8a24e5f8df4c42a5bff4a8349a2d33 is 175, key is test_row_0/A:col10/1733202018861/Put/seqid=0 2024-12-03T05:00:20,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202080123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:20,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202080132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:20,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742027_1203 (size=31903) 2024-12-03T05:00:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:00:20,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:00:20,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202080240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:20,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202080241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:20,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:00:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202080451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:20,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202080451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:20,471 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:20,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cceebd1e34e5483e8c21337cfb7fd7b8_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cceebd1e34e5483e8c21337cfb7fd7b8_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:20,488 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/65bdf1fccc984bc0830bf3a57fca22b8, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:20,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/65bdf1fccc984bc0830bf3a57fca22b8 is 175, key is test_row_0/A:col10/1733202020026/Put/seqid=0 2024-12-03T05:00:20,501 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/167b25140d85438a987f2e0cdde6804e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/167b25140d85438a987f2e0cdde6804e 2024-12-03T05:00:20,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742028_1204 (size=31255) 2024-12-03T05:00:20,509 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/B of b4ec74c11861541ec3b2d9523601012e into 167b25140d85438a987f2e0cdde6804e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:20,509 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:20,509 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/B, priority=13, startTime=1733202020025; duration=0sec 2024-12-03T05:00:20,509 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:20,509 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:B 2024-12-03T05:00:20,509 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:20,510 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:20,510 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/C is initiating minor compaction (all files) 2024-12-03T05:00:20,510 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/C in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,510 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a51c0745e5a144fe9ec562bfa6663c45, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/864b9fc3e77c4b4e95d29ef538a3b164, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/d4ea5719deb84b51905b509c32ed7c53] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=36.3 K 2024-12-03T05:00:20,511 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a51c0745e5a144fe9ec562bfa6663c45, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733202015770 2024-12-03T05:00:20,511 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 864b9fc3e77c4b4e95d29ef538a3b164, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733202017003 2024-12-03T05:00:20,512 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting d4ea5719deb84b51905b509c32ed7c53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733202018192 2024-12-03T05:00:20,529 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#C#compaction#168 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:20,530 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/a05084089a324a778eccf052481decfa is 50, key is test_row_0/C:col10/1733202018861/Put/seqid=0 2024-12-03T05:00:20,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742029_1205 (size=12949) 2024-12-03T05:00:20,553 INFO [master/32beccd76b35:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T05:00:20,553 INFO [master/32beccd76b35:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T05:00:20,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:00:20,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:20,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,578 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/ee8a24e5f8df4c42a5bff4a8349a2d33 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/ee8a24e5f8df4c42a5bff4a8349a2d33 2024-12-03T05:00:20,585 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/A of b4ec74c11861541ec3b2d9523601012e into ee8a24e5f8df4c42a5bff4a8349a2d33(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:20,585 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:20,585 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/A, priority=13, startTime=1733202020025; duration=0sec 2024-12-03T05:00:20,585 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:20,585 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:A 2024-12-03T05:00:20,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:00:20,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:20,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:00:20,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202080760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202080760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:20,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:00:20,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:20,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:20,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:20,903 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/65bdf1fccc984bc0830bf3a57fca22b8 2024-12-03T05:00:20,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/78dacd5285e243fdb5fa87867b7b1522 is 50, key is test_row_0/B:col10/1733202020026/Put/seqid=0 2024-12-03T05:00:20,948 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/a05084089a324a778eccf052481decfa as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a05084089a324a778eccf052481decfa 2024-12-03T05:00:20,956 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/C of b4ec74c11861541ec3b2d9523601012e into a05084089a324a778eccf052481decfa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:20,956 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:20,956 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/C, priority=13, startTime=1733202020028; duration=0sec 2024-12-03T05:00:20,957 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:20,957 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:C 2024-12-03T05:00:20,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742030_1206 (size=12301) 2024-12-03T05:00:20,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/78dacd5285e243fdb5fa87867b7b1522 2024-12-03T05:00:20,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/e7214ed9e4eb4099ac2d1ad9005b2f1d is 50, key is test_row_0/C:col10/1733202020026/Put/seqid=0 2024-12-03T05:00:20,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742031_1207 (size=12301) 2024-12-03T05:00:20,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/e7214ed9e4eb4099ac2d1ad9005b2f1d 2024-12-03T05:00:20,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/65bdf1fccc984bc0830bf3a57fca22b8 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/65bdf1fccc984bc0830bf3a57fca22b8 2024-12-03T05:00:21,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/65bdf1fccc984bc0830bf3a57fca22b8, entries=150, sequenceid=292, filesize=30.5 K 2024-12-03T05:00:21,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/78dacd5285e243fdb5fa87867b7b1522 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/78dacd5285e243fdb5fa87867b7b1522 2024-12-03T05:00:21,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/78dacd5285e243fdb5fa87867b7b1522, entries=150, sequenceid=292, filesize=12.0 K 2024-12-03T05:00:21,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:00:21,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/e7214ed9e4eb4099ac2d1ad9005b2f1d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e7214ed9e4eb4099ac2d1ad9005b2f1d 2024-12-03T05:00:21,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:21,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:21,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:21,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:21,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:21,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e7214ed9e4eb4099ac2d1ad9005b2f1d, entries=150, sequenceid=292, filesize=12.0 K 2024-12-03T05:00:21,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for b4ec74c11861541ec3b2d9523601012e in 996ms, sequenceid=292, compaction requested=false 2024-12-03T05:00:21,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:21,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:00:21,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:21,169 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-03T05:00:21,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b104bf23847942b88e117fa03ef2ae4e_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202020088/Put/seqid=0 2024-12-03T05:00:21,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742032_1208 (size=12454) 2024-12-03T05:00:21,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:21,195 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b104bf23847942b88e117fa03ef2ae4e_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b104bf23847942b88e117fa03ef2ae4e_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:21,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/a94c1dfec4a9467e832635d57a0a9702, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:21,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/a94c1dfec4a9467e832635d57a0a9702 is 175, key is test_row_0/A:col10/1733202020088/Put/seqid=0 2024-12-03T05:00:21,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742033_1209 (size=31255) 2024-12-03T05:00:21,216 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/a94c1dfec4a9467e832635d57a0a9702 2024-12-03T05:00:21,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/92ea9fb6a68b44a2b4b35aa886c259fb is 50, key is test_row_0/B:col10/1733202020088/Put/seqid=0 2024-12-03T05:00:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:21,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:21,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742034_1210 (size=12301) 2024-12-03T05:00:21,281 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/92ea9fb6a68b44a2b4b35aa886c259fb 2024-12-03T05:00:21,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/9d0b9e267919426b9ba3f8e6cfea85b6 is 50, key is test_row_0/C:col10/1733202020088/Put/seqid=0 2024-12-03T05:00:21,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202081287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:21,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202081291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:21,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742035_1211 (size=12301) 2024-12-03T05:00:21,311 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/9d0b9e267919426b9ba3f8e6cfea85b6 2024-12-03T05:00:21,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/a94c1dfec4a9467e832635d57a0a9702 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a94c1dfec4a9467e832635d57a0a9702 2024-12-03T05:00:21,324 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a94c1dfec4a9467e832635d57a0a9702, entries=150, sequenceid=321, filesize=30.5 K 2024-12-03T05:00:21,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/92ea9fb6a68b44a2b4b35aa886c259fb as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/92ea9fb6a68b44a2b4b35aa886c259fb 2024-12-03T05:00:21,335 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/92ea9fb6a68b44a2b4b35aa886c259fb, entries=150, sequenceid=321, filesize=12.0 K 2024-12-03T05:00:21,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/9d0b9e267919426b9ba3f8e6cfea85b6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/9d0b9e267919426b9ba3f8e6cfea85b6 2024-12-03T05:00:21,343 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/9d0b9e267919426b9ba3f8e6cfea85b6, entries=150, sequenceid=321, filesize=12.0 K 2024-12-03T05:00:21,345 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for b4ec74c11861541ec3b2d9523601012e in 175ms, sequenceid=321, compaction requested=true 2024-12-03T05:00:21,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:21,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:21,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-03T05:00:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-03T05:00:21,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-03T05:00:21,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7630 sec 2024-12-03T05:00:21,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.7700 sec 2024-12-03T05:00:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:21,404 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:21,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-03T05:00:21,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034376dd4242904b1d8d439ea8f728d6ca_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202021282/Put/seqid=0 2024-12-03T05:00:21,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742036_1212 (size=12454) 2024-12-03T05:00:21,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:21,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202081496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202081496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:21,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202081610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:21,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202081620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:00:21,731 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:21,731 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-03T05:00:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:00:21,734 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:21,736 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:21,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202081820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:21,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202081831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:00:21,878 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:21,886 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034376dd4242904b1d8d439ea8f728d6ca_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034376dd4242904b1d8d439ea8f728d6ca_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:21,887 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/462fc2b9e9da42a185ee531844b9ace2, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:21,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/462fc2b9e9da42a185ee531844b9ace2 is 175, key is test_row_0/A:col10/1733202021282/Put/seqid=0 2024-12-03T05:00:21,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T05:00:21,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:21,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:21,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:21,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:21,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:21,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742037_1213 (size=31255) 2024-12-03T05:00:22,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T05:00:22,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:22,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:22,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:22,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:22,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:00:22,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202082131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:22,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:22,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202082151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:22,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T05:00:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:22,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:22,298 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/462fc2b9e9da42a185ee531844b9ace2 2024-12-03T05:00:22,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/bca71f6b362f49668a02953d2248ef50 is 50, key is test_row_0/B:col10/1733202021282/Put/seqid=0 2024-12-03T05:00:22,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742038_1214 (size=12301) 2024-12-03T05:00:22,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/bca71f6b362f49668a02953d2248ef50 2024-12-03T05:00:22,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/593ecd9232354409bf3993faabf2180b is 50, key is test_row_0/C:col10/1733202021282/Put/seqid=0 2024-12-03T05:00:22,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742039_1215 (size=12301) 2024-12-03T05:00:22,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/593ecd9232354409bf3993faabf2180b 2024-12-03T05:00:22,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T05:00:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:22,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:00:22,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/462fc2b9e9da42a185ee531844b9ace2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/462fc2b9e9da42a185ee531844b9ace2 2024-12-03T05:00:22,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/462fc2b9e9da42a185ee531844b9ace2, entries=150, sequenceid=333, filesize=30.5 K 2024-12-03T05:00:22,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/bca71f6b362f49668a02953d2248ef50 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bca71f6b362f49668a02953d2248ef50 2024-12-03T05:00:22,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bca71f6b362f49668a02953d2248ef50, entries=150, sequenceid=333, filesize=12.0 K 2024-12-03T05:00:22,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/593ecd9232354409bf3993faabf2180b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/593ecd9232354409bf3993faabf2180b 2024-12-03T05:00:22,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/593ecd9232354409bf3993faabf2180b, entries=150, sequenceid=333, filesize=12.0 K 2024-12-03T05:00:22,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for b4ec74c11861541ec3b2d9523601012e in 981ms, sequenceid=333, compaction requested=true 2024-12-03T05:00:22,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:22,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:22,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:22,386 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:22,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:22,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:22,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:22,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:00:22,386 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:22,388 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:22,388 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/A is initiating minor compaction (all files) 2024-12-03T05:00:22,388 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/A in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:22,388 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/ee8a24e5f8df4c42a5bff4a8349a2d33, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/65bdf1fccc984bc0830bf3a57fca22b8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a94c1dfec4a9467e832635d57a0a9702, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/462fc2b9e9da42a185ee531844b9ace2] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=122.7 K 2024-12-03T05:00:22,388 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:22,388 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/ee8a24e5f8df4c42a5bff4a8349a2d33, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/65bdf1fccc984bc0830bf3a57fca22b8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a94c1dfec4a9467e832635d57a0a9702, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/462fc2b9e9da42a185ee531844b9ace2] 2024-12-03T05:00:22,389 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:22,389 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/B is initiating minor compaction (all files) 2024-12-03T05:00:22,389 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/B in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:22,389 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/167b25140d85438a987f2e0cdde6804e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/78dacd5285e243fdb5fa87867b7b1522, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/92ea9fb6a68b44a2b4b35aa886c259fb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bca71f6b362f49668a02953d2248ef50] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=48.7 K 2024-12-03T05:00:22,389 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 167b25140d85438a987f2e0cdde6804e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733202018192 2024-12-03T05:00:22,390 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee8a24e5f8df4c42a5bff4a8349a2d33, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733202018192 2024-12-03T05:00:22,390 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 78dacd5285e243fdb5fa87867b7b1522, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733202018876 2024-12-03T05:00:22,390 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65bdf1fccc984bc0830bf3a57fca22b8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733202018876 2024-12-03T05:00:22,391 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ea9fb6a68b44a2b4b35aa886c259fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733202020088 2024-12-03T05:00:22,391 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a94c1dfec4a9467e832635d57a0a9702, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733202020088 2024-12-03T05:00:22,391 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 462fc2b9e9da42a185ee531844b9ace2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733202021282 2024-12-03T05:00:22,392 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting bca71f6b362f49668a02953d2248ef50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733202021282 2024-12-03T05:00:22,408 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:22,409 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/1ef5616a472743829688c25b041daea6 is 50, key is test_row_0/B:col10/1733202021282/Put/seqid=0 2024-12-03T05:00:22,414 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:22,418 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033300888c581f4c06bdd013697334c99e_b4ec74c11861541ec3b2d9523601012e store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:22,420 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033300888c581f4c06bdd013697334c99e_b4ec74c11861541ec3b2d9523601012e, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:22,420 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033300888c581f4c06bdd013697334c99e_b4ec74c11861541ec3b2d9523601012e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:22,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742040_1216 (size=13085) 2024-12-03T05:00:22,430 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/1ef5616a472743829688c25b041daea6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/1ef5616a472743829688c25b041daea6 2024-12-03T05:00:22,439 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/B of b4ec74c11861541ec3b2d9523601012e into 1ef5616a472743829688c25b041daea6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:22,439 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:22,439 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/B, priority=12, startTime=1733202022386; duration=0sec 2024-12-03T05:00:22,439 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:22,439 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:B 2024-12-03T05:00:22,440 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:22,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742041_1217 (size=4469) 2024-12-03T05:00:22,442 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:22,442 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/C is initiating minor compaction (all files) 2024-12-03T05:00:22,442 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/C in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:22,442 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a05084089a324a778eccf052481decfa, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e7214ed9e4eb4099ac2d1ad9005b2f1d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/9d0b9e267919426b9ba3f8e6cfea85b6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/593ecd9232354409bf3993faabf2180b] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=48.7 K 2024-12-03T05:00:22,443 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a05084089a324a778eccf052481decfa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733202018192 2024-12-03T05:00:22,443 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting e7214ed9e4eb4099ac2d1ad9005b2f1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733202018876 2024-12-03T05:00:22,444 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d0b9e267919426b9ba3f8e6cfea85b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733202020088 2024-12-03T05:00:22,445 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 593ecd9232354409bf3993faabf2180b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733202021282 2024-12-03T05:00:22,449 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#A#compaction#178 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:22,449 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/acb63ca5a9b94b2180b3742be3f5a567 is 175, key is test_row_0/A:col10/1733202021282/Put/seqid=0 2024-12-03T05:00:22,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742042_1218 (size=32039) 2024-12-03T05:00:22,468 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/acb63ca5a9b94b2180b3742be3f5a567 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/acb63ca5a9b94b2180b3742be3f5a567 2024-12-03T05:00:22,474 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:22,475 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/2ec25b25be424b7ea3893cbcb2816ad7 is 50, key is test_row_0/C:col10/1733202021282/Put/seqid=0 2024-12-03T05:00:22,491 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/A of b4ec74c11861541ec3b2d9523601012e into acb63ca5a9b94b2180b3742be3f5a567(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:22,491 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:22,491 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/A, priority=12, startTime=1733202022386; duration=0sec 2024-12-03T05:00:22,491 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:22,491 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:A 2024-12-03T05:00:22,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742043_1219 (size=13085) 2024-12-03T05:00:22,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T05:00:22,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:22,511 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:00:22,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ece775b718254a2d9585a91735dc48ed_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202021482/Put/seqid=0 2024-12-03T05:00:22,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742044_1220 (size=12454) 2024-12-03T05:00:22,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:22,598 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ece775b718254a2d9585a91735dc48ed_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ece775b718254a2d9585a91735dc48ed_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:22,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/e3474e93557b44f682e164459d08e76f, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/e3474e93557b44f682e164459d08e76f is 175, key is test_row_0/A:col10/1733202021482/Put/seqid=0 2024-12-03T05:00:22,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742045_1221 (size=31255) 2024-12-03T05:00:22,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:22,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202082680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:22,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202082688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:22,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:22,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202082791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:22,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202082804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:00:22,905 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/2ec25b25be424b7ea3893cbcb2816ad7 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/2ec25b25be424b7ea3893cbcb2816ad7 2024-12-03T05:00:22,910 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/C of b4ec74c11861541ec3b2d9523601012e into 2ec25b25be424b7ea3893cbcb2816ad7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:22,910 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:22,910 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/C, priority=12, startTime=1733202022386; duration=0sec 2024-12-03T05:00:22,910 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:22,910 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:C 2024-12-03T05:00:23,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202083010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:23,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:23,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202083020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:23,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:23,024 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/e3474e93557b44f682e164459d08e76f 2024-12-03T05:00:23,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/c3731b96acc04cc38ef564827547c366 is 50, key is test_row_0/B:col10/1733202021482/Put/seqid=0 2024-12-03T05:00:23,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742046_1222 (size=12301) 2024-12-03T05:00:23,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202083320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:23,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202083330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:23,447 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/c3731b96acc04cc38ef564827547c366 2024-12-03T05:00:23,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/c4327a67ab4845b0b3054258744b7d32 is 50, key is test_row_0/C:col10/1733202021482/Put/seqid=0 2024-12-03T05:00:23,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742047_1223 (size=12301) 2024-12-03T05:00:23,501 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/c4327a67ab4845b0b3054258744b7d32 2024-12-03T05:00:23,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/e3474e93557b44f682e164459d08e76f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e3474e93557b44f682e164459d08e76f 2024-12-03T05:00:23,545 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e3474e93557b44f682e164459d08e76f, entries=150, sequenceid=360, filesize=30.5 K 2024-12-03T05:00:23,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/c3731b96acc04cc38ef564827547c366 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/c3731b96acc04cc38ef564827547c366 2024-12-03T05:00:23,570 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/c3731b96acc04cc38ef564827547c366, entries=150, sequenceid=360, filesize=12.0 K 2024-12-03T05:00:23,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/c4327a67ab4845b0b3054258744b7d32 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/c4327a67ab4845b0b3054258744b7d32 2024-12-03T05:00:23,591 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/c4327a67ab4845b0b3054258744b7d32, entries=150, sequenceid=360, filesize=12.0 K 2024-12-03T05:00:23,592 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for b4ec74c11861541ec3b2d9523601012e in 1081ms, sequenceid=360, compaction requested=false 2024-12-03T05:00:23,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:23,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:23,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-03T05:00:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-03T05:00:23,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-03T05:00:23,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-12-03T05:00:23,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8830 sec 2024-12-03T05:00:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:23,838 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:23,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-03T05:00:23,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038f1e0ba558fa42d4994c1fd7c4b75ca6_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202023834/Put/seqid=0 2024-12-03T05:00:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:00:23,881 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:23,882 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-03T05:00:23,885 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:23,886 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:23,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:00:23,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742048_1224 (size=14994) 2024-12-03T05:00:23,918 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:23,943 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038f1e0ba558fa42d4994c1fd7c4b75ca6_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038f1e0ba558fa42d4994c1fd7c4b75ca6_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:23,946 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/b041ea317a124b0f9faf858083994f1c, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:23,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/b041ea317a124b0f9faf858083994f1c is 175, key is test_row_0/A:col10/1733202023834/Put/seqid=0 2024-12-03T05:00:23,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202083954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:23,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202083958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:23,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742049_1225 (size=39949) 2024-12-03T05:00:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:00:24,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:00:24,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:24,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202084074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:24,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202084077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:00:24,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:00:24,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:24,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202084282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:24,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202084294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:24,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:24,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:00:24,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:24,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,393 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/b041ea317a124b0f9faf858083994f1c 2024-12-03T05:00:24,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/08063350ed2c4d928f77ce77be78cf19 is 50, key is test_row_0/B:col10/1733202023834/Put/seqid=0 2024-12-03T05:00:24,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742050_1226 (size=12301) 2024-12-03T05:00:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:00:24,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:00:24,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:24,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202084590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:24,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:24,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202084611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:24,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:00:24,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:24,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/08063350ed2c4d928f77ce77be78cf19 2024-12-03T05:00:24,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/783737a244ab4f109bdce4d6d9fa7816 is 50, key is test_row_0/C:col10/1733202023834/Put/seqid=0 2024-12-03T05:00:24,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:00:24,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:24,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:24,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:24,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742051_1227 (size=12301) 2024-12-03T05:00:25,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:00:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:25,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:00:25,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202085100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:25,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:25,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:25,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:25,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202085120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:25,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:00:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:25,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:25,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:25,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/783737a244ab4f109bdce4d6d9fa7816 2024-12-03T05:00:25,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/b041ea317a124b0f9faf858083994f1c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b041ea317a124b0f9faf858083994f1c 2024-12-03T05:00:25,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:00:25,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:25,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:25,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:25,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:25,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:25,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b041ea317a124b0f9faf858083994f1c, entries=200, sequenceid=374, filesize=39.0 K 2024-12-03T05:00:25,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/08063350ed2c4d928f77ce77be78cf19 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/08063350ed2c4d928f77ce77be78cf19 2024-12-03T05:00:25,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/08063350ed2c4d928f77ce77be78cf19, entries=150, sequenceid=374, filesize=12.0 K 2024-12-03T05:00:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/783737a244ab4f109bdce4d6d9fa7816 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/783737a244ab4f109bdce4d6d9fa7816 2024-12-03T05:00:25,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/783737a244ab4f109bdce4d6d9fa7816, entries=150, sequenceid=374, filesize=12.0 K 2024-12-03T05:00:25,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for b4ec74c11861541ec3b2d9523601012e in 1530ms, sequenceid=374, compaction requested=true 2024-12-03T05:00:25,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:25,368 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:25,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:25,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:25,377 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:25,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:25,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:25,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:25,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:25,378 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:25,379 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/A is initiating minor compaction (all files) 2024-12-03T05:00:25,379 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/A in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:25,379 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/acb63ca5a9b94b2180b3742be3f5a567, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e3474e93557b44f682e164459d08e76f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b041ea317a124b0f9faf858083994f1c] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=100.8 K 2024-12-03T05:00:25,379 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:25,379 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/acb63ca5a9b94b2180b3742be3f5a567, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e3474e93557b44f682e164459d08e76f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b041ea317a124b0f9faf858083994f1c] 2024-12-03T05:00:25,385 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting acb63ca5a9b94b2180b3742be3f5a567, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733202021282 2024-12-03T05:00:25,386 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:25,386 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/B is initiating minor compaction (all files) 2024-12-03T05:00:25,386 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/B in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:25,386 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/1ef5616a472743829688c25b041daea6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/c3731b96acc04cc38ef564827547c366, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/08063350ed2c4d928f77ce77be78cf19] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=36.8 K 2024-12-03T05:00:25,386 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting e3474e93557b44f682e164459d08e76f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733202021476 2024-12-03T05:00:25,386 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ef5616a472743829688c25b041daea6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733202021282 2024-12-03T05:00:25,387 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting b041ea317a124b0f9faf858083994f1c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733202022670 2024-12-03T05:00:25,387 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3731b96acc04cc38ef564827547c366, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733202021476 2024-12-03T05:00:25,388 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08063350ed2c4d928f77ce77be78cf19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733202022670 2024-12-03T05:00:25,410 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:25,411 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/b173cc059c6f47e38a5dc90314454741 is 50, key is test_row_0/B:col10/1733202023834/Put/seqid=0 2024-12-03T05:00:25,414 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:25,422 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120391f91bf645e84d2ca754803671865b4f_b4ec74c11861541ec3b2d9523601012e store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:25,424 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120391f91bf645e84d2ca754803671865b4f_b4ec74c11861541ec3b2d9523601012e, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:25,424 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120391f91bf645e84d2ca754803671865b4f_b4ec74c11861541ec3b2d9523601012e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:25,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742052_1228 (size=13187) 2024-12-03T05:00:25,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:00:25,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:25,481 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-03T05:00:25,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742053_1229 (size=4469) 2024-12-03T05:00:25,491 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#A#compaction#187 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:25,491 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/46ab0e32bf5241f19c9cccf0f9ac6270 is 175, key is test_row_0/A:col10/1733202023834/Put/seqid=0 2024-12-03T05:00:25,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031d88381f786b4add8953942af67c4a9d_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202023949/Put/seqid=0 2024-12-03T05:00:25,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742054_1230 (size=32141) 2024-12-03T05:00:25,552 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/46ab0e32bf5241f19c9cccf0f9ac6270 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/46ab0e32bf5241f19c9cccf0f9ac6270 2024-12-03T05:00:25,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742055_1231 (size=12454) 2024-12-03T05:00:25,571 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/A of b4ec74c11861541ec3b2d9523601012e into 46ab0e32bf5241f19c9cccf0f9ac6270(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:25,571 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:25,571 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/A, priority=13, startTime=1733202025368; duration=0sec 2024-12-03T05:00:25,571 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:25,571 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:A 2024-12-03T05:00:25,571 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:25,574 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:25,574 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/C is initiating minor compaction (all files) 2024-12-03T05:00:25,574 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/C in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:25,574 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/2ec25b25be424b7ea3893cbcb2816ad7, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/c4327a67ab4845b0b3054258744b7d32, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/783737a244ab4f109bdce4d6d9fa7816] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=36.8 K 2024-12-03T05:00:25,575 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec25b25be424b7ea3893cbcb2816ad7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733202021282 2024-12-03T05:00:25,575 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c4327a67ab4845b0b3054258744b7d32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733202021476 2024-12-03T05:00:25,576 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 783737a244ab4f109bdce4d6d9fa7816, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733202022670 2024-12-03T05:00:25,591 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:25,592 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/a11f79b01c504c04af24ca6f019e8945 is 50, key is test_row_0/C:col10/1733202023834/Put/seqid=0 2024-12-03T05:00:25,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742056_1232 (size=13187) 2024-12-03T05:00:25,623 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/a11f79b01c504c04af24ca6f019e8945 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a11f79b01c504c04af24ca6f019e8945 2024-12-03T05:00:25,636 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/C of b4ec74c11861541ec3b2d9523601012e into a11f79b01c504c04af24ca6f019e8945(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:25,636 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:25,636 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/C, priority=13, startTime=1733202025378; duration=0sec 2024-12-03T05:00:25,636 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:25,636 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:C 2024-12-03T05:00:25,876 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/b173cc059c6f47e38a5dc90314454741 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/b173cc059c6f47e38a5dc90314454741 2024-12-03T05:00:25,883 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/B of b4ec74c11861541ec3b2d9523601012e into b173cc059c6f47e38a5dc90314454741(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:25,883 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:25,883 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/B, priority=13, startTime=1733202025377; duration=0sec 2024-12-03T05:00:25,883 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:25,884 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:B 2024-12-03T05:00:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:25,962 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031d88381f786b4add8953942af67c4a9d_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031d88381f786b4add8953942af67c4a9d_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/f42bf052fb374b228e895a787be86a3e, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/f42bf052fb374b228e895a787be86a3e is 175, key is test_row_0/A:col10/1733202023949/Put/seqid=0 2024-12-03T05:00:26,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742057_1233 (size=31255) 2024-12-03T05:00:26,002 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/f42bf052fb374b228e895a787be86a3e 2024-12-03T05:00:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/4e00f3d103ce41738b659b145d840fb6 is 50, key is test_row_0/B:col10/1733202023949/Put/seqid=0 2024-12-03T05:00:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:00:26,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742058_1234 (size=12301) 2024-12-03T05:00:26,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. as already flushing 2024-12-03T05:00:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:26,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202086166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:26,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202086167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:26,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202086271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:26,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202086280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:26,478 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/4e00f3d103ce41738b659b145d840fb6 2024-12-03T05:00:26,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57428 deadline: 1733202086480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:26,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57418 deadline: 1733202086491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:00:26,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/8acbfa260cac4597a7d5d16a399bc491 is 50, key is test_row_0/C:col10/1733202023949/Put/seqid=0 2024-12-03T05:00:26,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742059_1235 (size=12301) 2024-12-03T05:00:26,548 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/8acbfa260cac4597a7d5d16a399bc491 2024-12-03T05:00:26,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/f42bf052fb374b228e895a787be86a3e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/f42bf052fb374b228e895a787be86a3e 2024-12-03T05:00:26,591 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/f42bf052fb374b228e895a787be86a3e, entries=150, sequenceid=398, filesize=30.5 K 2024-12-03T05:00:26,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/4e00f3d103ce41738b659b145d840fb6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/4e00f3d103ce41738b659b145d840fb6 2024-12-03T05:00:26,610 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/4e00f3d103ce41738b659b145d840fb6, entries=150, sequenceid=398, filesize=12.0 K 2024-12-03T05:00:26,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/8acbfa260cac4597a7d5d16a399bc491 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8acbfa260cac4597a7d5d16a399bc491 2024-12-03T05:00:26,640 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8acbfa260cac4597a7d5d16a399bc491, entries=150, sequenceid=398, filesize=12.0 K 2024-12-03T05:00:26,649 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for b4ec74c11861541ec3b2d9523601012e in 1168ms, sequenceid=398, compaction requested=false 2024-12-03T05:00:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:26,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-03T05:00:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-03T05:00:26,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-03T05:00:26,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7870 sec 2024-12-03T05:00:26,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.8130 sec 2024-12-03T05:00:26,784 INFO [Thread-631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-631. 2024-12-03T05:00:26,784 DEBUG [Thread-631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:26,784 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,784 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:26,787 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-12-03T05:00:26,787 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:26,787 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,787 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:26,788 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-12-03T05:00:26,788 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:26,788 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,788 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:26,789 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-12-03T05:00:26,789 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:26,789 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,789 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:26,804 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:26,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-03T05:00:26,805 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-12-03T05:00:26,805 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:26,805 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,805 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:26,807 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-12-03T05:00:26,807 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:26,807 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,807 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:26,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:26,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ed0fabd3159f45f286021aa91eaea8e9_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202026164/Put/seqid=0 2024-12-03T05:00:26,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742060_1236 (size=12454) 2024-12-03T05:00:27,222 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:27,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ed0fabd3159f45f286021aa91eaea8e9_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ed0fabd3159f45f286021aa91eaea8e9_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:27,229 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/a51e32b236d745a4ba176fac90be0dc7, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:27,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/a51e32b236d745a4ba176fac90be0dc7 is 175, key is test_row_0/A:col10/1733202026164/Put/seqid=0 2024-12-03T05:00:27,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742061_1237 (size=31255) 2024-12-03T05:00:27,635 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/a51e32b236d745a4ba176fac90be0dc7 2024-12-03T05:00:27,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/139938f7e3484b9fbb5796349b2cd8f7 is 50, key is test_row_0/B:col10/1733202026164/Put/seqid=0 2024-12-03T05:00:27,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742062_1238 (size=12301) 2024-12-03T05:00:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:00:28,042 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:28,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/139938f7e3484b9fbb5796349b2cd8f7 2024-12-03T05:00:28,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/81a38247ea6a4d388c24a25b0cb414dc is 50, key is test_row_0/C:col10/1733202026164/Put/seqid=0 2024-12-03T05:00:28,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742063_1239 (size=12301) 2024-12-03T05:00:28,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/81a38247ea6a4d388c24a25b0cb414dc 2024-12-03T05:00:28,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/a51e32b236d745a4ba176fac90be0dc7 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a51e32b236d745a4ba176fac90be0dc7 2024-12-03T05:00:28,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a51e32b236d745a4ba176fac90be0dc7, entries=150, sequenceid=416, filesize=30.5 K 2024-12-03T05:00:28,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/139938f7e3484b9fbb5796349b2cd8f7 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/139938f7e3484b9fbb5796349b2cd8f7 2024-12-03T05:00:28,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/139938f7e3484b9fbb5796349b2cd8f7, entries=150, sequenceid=416, filesize=12.0 K 2024-12-03T05:00:28,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/81a38247ea6a4d388c24a25b0cb414dc as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/81a38247ea6a4d388c24a25b0cb414dc 2024-12-03T05:00:28,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/81a38247ea6a4d388c24a25b0cb414dc, entries=150, sequenceid=416, filesize=12.0 K 2024-12-03T05:00:28,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for b4ec74c11861541ec3b2d9523601012e in 1677ms, sequenceid=416, compaction requested=true 2024-12-03T05:00:28,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:28,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:28,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:28,481 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:28,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:28,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:28,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4ec74c11861541ec3b2d9523601012e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:28,481 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:28,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:28,482 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:28,482 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:28,482 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/A is initiating minor compaction (all files) 2024-12-03T05:00:28,482 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/B is initiating minor compaction (all files) 2024-12-03T05:00:28,482 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/A in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:28,482 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/B in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:28,483 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/46ab0e32bf5241f19c9cccf0f9ac6270, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/f42bf052fb374b228e895a787be86a3e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a51e32b236d745a4ba176fac90be0dc7] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=92.4 K 2024-12-03T05:00:28,483 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/b173cc059c6f47e38a5dc90314454741, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/4e00f3d103ce41738b659b145d840fb6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/139938f7e3484b9fbb5796349b2cd8f7] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=36.9 K 2024-12-03T05:00:28,483 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:28,483 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/46ab0e32bf5241f19c9cccf0f9ac6270, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/f42bf052fb374b228e895a787be86a3e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a51e32b236d745a4ba176fac90be0dc7] 2024-12-03T05:00:28,483 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting b173cc059c6f47e38a5dc90314454741, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733202022670 2024-12-03T05:00:28,483 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46ab0e32bf5241f19c9cccf0f9ac6270, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733202022670 2024-12-03T05:00:28,483 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e00f3d103ce41738b659b145d840fb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733202023927 2024-12-03T05:00:28,483 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f42bf052fb374b228e895a787be86a3e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733202023927 2024-12-03T05:00:28,484 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 139938f7e3484b9fbb5796349b2cd8f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733202026164 2024-12-03T05:00:28,484 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a51e32b236d745a4ba176fac90be0dc7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733202026164 2024-12-03T05:00:28,492 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:28,492 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/f4c066e2d6c34dc3adf83c9e684133a9 is 50, key is test_row_0/B:col10/1733202026164/Put/seqid=0 2024-12-03T05:00:28,496 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:28,500 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203a98bcab372a34b3fbaaebe49a5e915f6_b4ec74c11861541ec3b2d9523601012e store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:28,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742064_1240 (size=13289) 2024-12-03T05:00:28,509 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/f4c066e2d6c34dc3adf83c9e684133a9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/f4c066e2d6c34dc3adf83c9e684133a9 2024-12-03T05:00:28,515 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/B of b4ec74c11861541ec3b2d9523601012e into f4c066e2d6c34dc3adf83c9e684133a9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:28,516 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:28,516 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/B, priority=13, startTime=1733202028481; duration=0sec 2024-12-03T05:00:28,517 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:28,517 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:B 2024-12-03T05:00:28,518 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:28,520 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:28,520 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): b4ec74c11861541ec3b2d9523601012e/C is initiating minor compaction (all files) 2024-12-03T05:00:28,520 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4ec74c11861541ec3b2d9523601012e/C in TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:28,520 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a11f79b01c504c04af24ca6f019e8945, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8acbfa260cac4597a7d5d16a399bc491, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/81a38247ea6a4d388c24a25b0cb414dc] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp, totalSize=36.9 K 2024-12-03T05:00:28,520 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a11f79b01c504c04af24ca6f019e8945, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733202022670 2024-12-03T05:00:28,521 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 8acbfa260cac4597a7d5d16a399bc491, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733202023927 2024-12-03T05:00:28,521 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 81a38247ea6a4d388c24a25b0cb414dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733202026164 2024-12-03T05:00:28,528 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203a98bcab372a34b3fbaaebe49a5e915f6_b4ec74c11861541ec3b2d9523601012e, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:28,528 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a98bcab372a34b3fbaaebe49a5e915f6_b4ec74c11861541ec3b2d9523601012e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:28,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742065_1241 (size=4469) 2024-12-03T05:00:28,543 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#C#compaction#197 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:28,543 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/e64204c7e0f844b991e511116b6019dd is 50, key is test_row_0/C:col10/1733202026164/Put/seqid=0 2024-12-03T05:00:28,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742066_1242 (size=13289) 2024-12-03T05:00:28,664 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-12-03T05:00:28,664 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:28,664 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:28,665 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:28,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:28,694 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-12-03T05:00:28,694 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:28,694 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:28,694 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:28,696 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:28,762 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-12-03T05:00:28,763 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:28,763 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:28,763 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:28,763 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2330 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2354 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1079 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3236 rows 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1070 2024-12-03T05:00:28,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3206 rows 2024-12-03T05:00:28,763 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:00:28,763 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:00:28,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:28,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:28,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:28,765 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:00:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:00:28,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202028768"}]},"ts":"1733202028768"} 2024-12-03T05:00:28,770 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:00:28,770 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:00:28,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:00:28,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, UNASSIGN}] 2024-12-03T05:00:28,773 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, UNASSIGN 2024-12-03T05:00:28,774 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=b4ec74c11861541ec3b2d9523601012e, regionState=CLOSING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:00:28,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, UNASSIGN because future has completed 2024-12-03T05:00:28,776 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:00:28,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:00:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:00:28,929 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:28,929 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:00:28,929 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing b4ec74c11861541ec3b2d9523601012e, disabling compactions & flushes 2024-12-03T05:00:28,929 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:28,943 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4ec74c11861541ec3b2d9523601012e#A#compaction#196 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:28,943 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/907657dfc55c46089c4640617256032d is 175, key is test_row_0/A:col10/1733202026164/Put/seqid=0 2024-12-03T05:00:28,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742067_1243 (size=32243) 2024-12-03T05:00:28,954 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/e64204c7e0f844b991e511116b6019dd as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e64204c7e0f844b991e511116b6019dd 2024-12-03T05:00:28,960 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/C of b4ec74c11861541ec3b2d9523601012e into e64204c7e0f844b991e511116b6019dd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:28,960 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:28,961 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/C, priority=13, startTime=1733202028481; duration=0sec 2024-12-03T05:00:28,961 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:28,961 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:C 2024-12-03T05:00:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:00:29,353 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/907657dfc55c46089c4640617256032d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/907657dfc55c46089c4640617256032d 2024-12-03T05:00:29,358 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4ec74c11861541ec3b2d9523601012e/A of b4ec74c11861541ec3b2d9523601012e into 907657dfc55c46089c4640617256032d(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:29,358 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4ec74c11861541ec3b2d9523601012e: 2024-12-03T05:00:29,358 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., storeName=b4ec74c11861541ec3b2d9523601012e/A, priority=13, startTime=1733202028481; duration=0sec 2024-12-03T05:00:29,358 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:29,358 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:29,358 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4ec74c11861541ec3b2d9523601012e:A 2024-12-03T05:00:29,358 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:29,359 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. after waiting 0 ms 2024-12-03T05:00:29,359 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:29,359 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing b4ec74c11861541ec3b2d9523601012e 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-12-03T05:00:29,365 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034398c43201c244ce8b6670fea5d32d3c_b4ec74c11861541ec3b2d9523601012e is 50, key is test_row_0/A:col10/1733202028761/Put/seqid=0 2024-12-03T05:00:29,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742068_1244 (size=9914) 2024-12-03T05:00:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:00:29,770 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:29,774 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034398c43201c244ce8b6670fea5d32d3c_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034398c43201c244ce8b6670fea5d32d3c_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:29,775 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/fa57ad0806304176931880d3a89ca0d3, store: [table=TestAcidGuarantees family=A region=b4ec74c11861541ec3b2d9523601012e] 2024-12-03T05:00:29,776 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/fa57ad0806304176931880d3a89ca0d3 is 175, key is test_row_0/A:col10/1733202028761/Put/seqid=0 2024-12-03T05:00:29,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742069_1245 (size=22561) 2024-12-03T05:00:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:00:30,181 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=425, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/fa57ad0806304176931880d3a89ca0d3 2024-12-03T05:00:30,189 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/7310f07715e84d74a51b6eb7d9200c95 is 50, key is test_row_0/B:col10/1733202028761/Put/seqid=0 2024-12-03T05:00:30,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742070_1246 (size=9857) 2024-12-03T05:00:30,595 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/7310f07715e84d74a51b6eb7d9200c95 2024-12-03T05:00:30,602 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/c12b82ca531a4bc08bce5747d7e74513 is 50, key is test_row_0/C:col10/1733202028761/Put/seqid=0 2024-12-03T05:00:30,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742071_1247 (size=9857) 2024-12-03T05:00:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:00:31,006 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/c12b82ca531a4bc08bce5747d7e74513 2024-12-03T05:00:31,011 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/A/fa57ad0806304176931880d3a89ca0d3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/fa57ad0806304176931880d3a89ca0d3 2024-12-03T05:00:31,015 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/fa57ad0806304176931880d3a89ca0d3, entries=100, sequenceid=425, filesize=22.0 K 2024-12-03T05:00:31,015 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/B/7310f07715e84d74a51b6eb7d9200c95 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/7310f07715e84d74a51b6eb7d9200c95 2024-12-03T05:00:31,019 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/7310f07715e84d74a51b6eb7d9200c95, entries=100, sequenceid=425, filesize=9.6 K 2024-12-03T05:00:31,020 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/.tmp/C/c12b82ca531a4bc08bce5747d7e74513 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/c12b82ca531a4bc08bce5747d7e74513 2024-12-03T05:00:31,024 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/c12b82ca531a4bc08bce5747d7e74513, entries=100, sequenceid=425, filesize=9.6 K 2024-12-03T05:00:31,025 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for b4ec74c11861541ec3b2d9523601012e in 1665ms, sequenceid=425, compaction requested=false 2024-12-03T05:00:31,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d32c093578d94c24bba02c5562d91a9d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d2f9ad92af084427a6773ac067f70fea, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e6fb72d41fae4e3e9a641c465cab50a5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb191d313ea8488ba6ab479103b79ce9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b85920c6b7884e87b47c0a55c0f94300, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1197af8f279b445daa73f46c53294344, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1a4b21138fcb4c9f93eda79ec7248619, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/94bd155acec44045b9fba60c98cc78aa, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/64be5146cec84b20a06e32944429a2f3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb04a80c8f4b449386c23ea3df163a0e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/0dd71e170ca04a17b714de7dadce6af5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/6654c78bda6d4f9d8f80fc24a776daae, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/01d145b0557b4c9db30d78d298cfd171, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1392dcdae5da4ebeaadc3feddf7c54f2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/5eec0115e66443a0871c346c09d77974, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a9749b8aafc849249d6807726fa27ec2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/32ef627a49c943e69363f7485bc961e8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/fd888b90e23e47cb81cff0b7164b6aa0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1ff21c47d1dd41009f899185a3301eb1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/ee8a24e5f8df4c42a5bff4a8349a2d33, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/65bdf1fccc984bc0830bf3a57fca22b8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a94c1dfec4a9467e832635d57a0a9702, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/acb63ca5a9b94b2180b3742be3f5a567, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/462fc2b9e9da42a185ee531844b9ace2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e3474e93557b44f682e164459d08e76f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b041ea317a124b0f9faf858083994f1c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/46ab0e32bf5241f19c9cccf0f9ac6270, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/f42bf052fb374b228e895a787be86a3e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a51e32b236d745a4ba176fac90be0dc7] to archive 2024-12-03T05:00:31,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:00:31,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d32c093578d94c24bba02c5562d91a9d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d32c093578d94c24bba02c5562d91a9d 2024-12-03T05:00:31,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d2f9ad92af084427a6773ac067f70fea to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/d2f9ad92af084427a6773ac067f70fea 2024-12-03T05:00:31,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e6fb72d41fae4e3e9a641c465cab50a5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e6fb72d41fae4e3e9a641c465cab50a5 2024-12-03T05:00:31,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb191d313ea8488ba6ab479103b79ce9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb191d313ea8488ba6ab479103b79ce9 2024-12-03T05:00:31,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b85920c6b7884e87b47c0a55c0f94300 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b85920c6b7884e87b47c0a55c0f94300 2024-12-03T05:00:31,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1197af8f279b445daa73f46c53294344 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1197af8f279b445daa73f46c53294344 2024-12-03T05:00:31,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1a4b21138fcb4c9f93eda79ec7248619 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1a4b21138fcb4c9f93eda79ec7248619 2024-12-03T05:00:31,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/94bd155acec44045b9fba60c98cc78aa to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/94bd155acec44045b9fba60c98cc78aa 2024-12-03T05:00:31,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/64be5146cec84b20a06e32944429a2f3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/64be5146cec84b20a06e32944429a2f3 2024-12-03T05:00:31,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb04a80c8f4b449386c23ea3df163a0e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/eb04a80c8f4b449386c23ea3df163a0e 2024-12-03T05:00:31,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/0dd71e170ca04a17b714de7dadce6af5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/0dd71e170ca04a17b714de7dadce6af5 2024-12-03T05:00:31,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/6654c78bda6d4f9d8f80fc24a776daae to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/6654c78bda6d4f9d8f80fc24a776daae 2024-12-03T05:00:31,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/01d145b0557b4c9db30d78d298cfd171 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/01d145b0557b4c9db30d78d298cfd171 2024-12-03T05:00:31,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1392dcdae5da4ebeaadc3feddf7c54f2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1392dcdae5da4ebeaadc3feddf7c54f2 2024-12-03T05:00:31,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/5eec0115e66443a0871c346c09d77974 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/5eec0115e66443a0871c346c09d77974 2024-12-03T05:00:31,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a9749b8aafc849249d6807726fa27ec2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a9749b8aafc849249d6807726fa27ec2 2024-12-03T05:00:31,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/32ef627a49c943e69363f7485bc961e8 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/32ef627a49c943e69363f7485bc961e8 2024-12-03T05:00:31,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/fd888b90e23e47cb81cff0b7164b6aa0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/fd888b90e23e47cb81cff0b7164b6aa0 2024-12-03T05:00:31,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1ff21c47d1dd41009f899185a3301eb1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/1ff21c47d1dd41009f899185a3301eb1 2024-12-03T05:00:31,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/ee8a24e5f8df4c42a5bff4a8349a2d33 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/ee8a24e5f8df4c42a5bff4a8349a2d33 2024-12-03T05:00:31,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/65bdf1fccc984bc0830bf3a57fca22b8 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/65bdf1fccc984bc0830bf3a57fca22b8 2024-12-03T05:00:31,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a94c1dfec4a9467e832635d57a0a9702 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a94c1dfec4a9467e832635d57a0a9702 2024-12-03T05:00:31,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/acb63ca5a9b94b2180b3742be3f5a567 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/acb63ca5a9b94b2180b3742be3f5a567 2024-12-03T05:00:31,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/462fc2b9e9da42a185ee531844b9ace2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/462fc2b9e9da42a185ee531844b9ace2 2024-12-03T05:00:31,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e3474e93557b44f682e164459d08e76f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/e3474e93557b44f682e164459d08e76f 2024-12-03T05:00:31,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b041ea317a124b0f9faf858083994f1c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/b041ea317a124b0f9faf858083994f1c 2024-12-03T05:00:31,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/46ab0e32bf5241f19c9cccf0f9ac6270 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/46ab0e32bf5241f19c9cccf0f9ac6270 2024-12-03T05:00:31,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/f42bf052fb374b228e895a787be86a3e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/f42bf052fb374b228e895a787be86a3e 2024-12-03T05:00:31,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a51e32b236d745a4ba176fac90be0dc7 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/a51e32b236d745a4ba176fac90be0dc7 2024-12-03T05:00:31,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/21206a532e384a1aabc18ba1876f6b34, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/464090caf3fe4e279c08c00826594c19, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/115ac88266914c459542892749957af0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/03e232ef23c14195890fbe13fa36b070, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/a0d142eaa2cc426a9299cbff29acd5fa, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/036b91d59f164a58b0c84e037d3ebd04, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/f1d35452d0ce403abf6d5d4ebb92dc51, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/13efb4d1e53348d6a210402dc4502f03, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/281c3f46c3af45ebbd9065fac9b3149b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/1ae3d7a0a6f54908af1bc73d64db9671, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bcb743ac1d8b482c90e1f8ae937b9448, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/312f9eb69c34483782f306d12228c93f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/dd028eb28dce43efbfdcdc4d97c4dd03, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/27730a0a7ea34083be502641c4cd0a2d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cfe56801b5f149798a86a1e0ca06fbbf, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/66634bc3972b4304b4f7b87f02832650, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/525729aa63ab427394fa923d7289268e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cb4c6723be8c4bd5a16d24eea2f62a31, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/167b25140d85438a987f2e0cdde6804e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/b5149458c7aa49cc9c670d3dbe984a10, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/78dacd5285e243fdb5fa87867b7b1522, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/92ea9fb6a68b44a2b4b35aa886c259fb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/1ef5616a472743829688c25b041daea6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bca71f6b362f49668a02953d2248ef50, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/c3731b96acc04cc38ef564827547c366, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/b173cc059c6f47e38a5dc90314454741, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/08063350ed2c4d928f77ce77be78cf19, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/4e00f3d103ce41738b659b145d840fb6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/139938f7e3484b9fbb5796349b2cd8f7] to archive 2024-12-03T05:00:31,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:00:31,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/21206a532e384a1aabc18ba1876f6b34 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/21206a532e384a1aabc18ba1876f6b34 2024-12-03T05:00:31,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/464090caf3fe4e279c08c00826594c19 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/464090caf3fe4e279c08c00826594c19 2024-12-03T05:00:31,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/115ac88266914c459542892749957af0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/115ac88266914c459542892749957af0 2024-12-03T05:00:31,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/03e232ef23c14195890fbe13fa36b070 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/03e232ef23c14195890fbe13fa36b070 2024-12-03T05:00:31,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/a0d142eaa2cc426a9299cbff29acd5fa to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/a0d142eaa2cc426a9299cbff29acd5fa 2024-12-03T05:00:31,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/036b91d59f164a58b0c84e037d3ebd04 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/036b91d59f164a58b0c84e037d3ebd04 2024-12-03T05:00:31,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/f1d35452d0ce403abf6d5d4ebb92dc51 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/f1d35452d0ce403abf6d5d4ebb92dc51 2024-12-03T05:00:31,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/13efb4d1e53348d6a210402dc4502f03 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/13efb4d1e53348d6a210402dc4502f03 2024-12-03T05:00:31,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/281c3f46c3af45ebbd9065fac9b3149b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/281c3f46c3af45ebbd9065fac9b3149b 2024-12-03T05:00:31,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/1ae3d7a0a6f54908af1bc73d64db9671 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/1ae3d7a0a6f54908af1bc73d64db9671 2024-12-03T05:00:31,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bcb743ac1d8b482c90e1f8ae937b9448 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bcb743ac1d8b482c90e1f8ae937b9448 2024-12-03T05:00:31,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/312f9eb69c34483782f306d12228c93f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/312f9eb69c34483782f306d12228c93f 2024-12-03T05:00:31,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/dd028eb28dce43efbfdcdc4d97c4dd03 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/dd028eb28dce43efbfdcdc4d97c4dd03 2024-12-03T05:00:31,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/27730a0a7ea34083be502641c4cd0a2d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/27730a0a7ea34083be502641c4cd0a2d 2024-12-03T05:00:31,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cfe56801b5f149798a86a1e0ca06fbbf to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cfe56801b5f149798a86a1e0ca06fbbf 2024-12-03T05:00:31,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/66634bc3972b4304b4f7b87f02832650 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/66634bc3972b4304b4f7b87f02832650 2024-12-03T05:00:31,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/525729aa63ab427394fa923d7289268e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/525729aa63ab427394fa923d7289268e 2024-12-03T05:00:31,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cb4c6723be8c4bd5a16d24eea2f62a31 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/cb4c6723be8c4bd5a16d24eea2f62a31 2024-12-03T05:00:31,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/167b25140d85438a987f2e0cdde6804e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/167b25140d85438a987f2e0cdde6804e 2024-12-03T05:00:31,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/b5149458c7aa49cc9c670d3dbe984a10 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/b5149458c7aa49cc9c670d3dbe984a10 2024-12-03T05:00:31,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/78dacd5285e243fdb5fa87867b7b1522 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/78dacd5285e243fdb5fa87867b7b1522 2024-12-03T05:00:31,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/92ea9fb6a68b44a2b4b35aa886c259fb to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/92ea9fb6a68b44a2b4b35aa886c259fb 2024-12-03T05:00:31,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/1ef5616a472743829688c25b041daea6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/1ef5616a472743829688c25b041daea6 2024-12-03T05:00:31,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bca71f6b362f49668a02953d2248ef50 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/bca71f6b362f49668a02953d2248ef50 2024-12-03T05:00:31,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/c3731b96acc04cc38ef564827547c366 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/c3731b96acc04cc38ef564827547c366 2024-12-03T05:00:31,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/b173cc059c6f47e38a5dc90314454741 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/b173cc059c6f47e38a5dc90314454741 2024-12-03T05:00:31,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/08063350ed2c4d928f77ce77be78cf19 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/08063350ed2c4d928f77ce77be78cf19 2024-12-03T05:00:31,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/4e00f3d103ce41738b659b145d840fb6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/4e00f3d103ce41738b659b145d840fb6 2024-12-03T05:00:31,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/139938f7e3484b9fbb5796349b2cd8f7 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/139938f7e3484b9fbb5796349b2cd8f7 2024-12-03T05:00:31,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/487620b102df4aa2be2b0f57011cae13, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ada920a11ae54afcb3f6fe933f9fed18, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/67c30e1adbc64292a16c82626edf8e98, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ab052b931cab4f6fb727272d86f59335, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/7e78c0da0c654f0f98861aae0f6f0602, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6f4d19d52bc948f1aa40f3408f0773a5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/4c83ab82746641f78159edd42e0fd4f1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a70e5f3c436548dfaa0ead0eabb701b8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e71ca8a682074deebbd23ec36ee2fe5b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ecbf73f84e1e4a029bdb63e9af455736, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6c3f36ec4e9a40a4ab245c6a63c2931d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8774b27be3eb47d8b6f408e664a26ad0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8f62f5c9cb6f4e5eb3964e7b99400688, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ddf4df2dd1d541ed827029b6760ff07b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/bad8198fcb1045c7aa42f9bda5163be4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a51c0745e5a144fe9ec562bfa6663c45, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/93d45f622da44eeea159a7eebc1c5b9a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/864b9fc3e77c4b4e95d29ef538a3b164, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a05084089a324a778eccf052481decfa, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/d4ea5719deb84b51905b509c32ed7c53, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e7214ed9e4eb4099ac2d1ad9005b2f1d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/9d0b9e267919426b9ba3f8e6cfea85b6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/2ec25b25be424b7ea3893cbcb2816ad7, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/593ecd9232354409bf3993faabf2180b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/c4327a67ab4845b0b3054258744b7d32, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a11f79b01c504c04af24ca6f019e8945, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/783737a244ab4f109bdce4d6d9fa7816, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8acbfa260cac4597a7d5d16a399bc491, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/81a38247ea6a4d388c24a25b0cb414dc] to archive 2024-12-03T05:00:31,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:00:31,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/487620b102df4aa2be2b0f57011cae13 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/487620b102df4aa2be2b0f57011cae13 2024-12-03T05:00:31,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ada920a11ae54afcb3f6fe933f9fed18 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ada920a11ae54afcb3f6fe933f9fed18 2024-12-03T05:00:31,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/67c30e1adbc64292a16c82626edf8e98 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/67c30e1adbc64292a16c82626edf8e98 2024-12-03T05:00:31,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ab052b931cab4f6fb727272d86f59335 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ab052b931cab4f6fb727272d86f59335 2024-12-03T05:00:31,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/7e78c0da0c654f0f98861aae0f6f0602 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/7e78c0da0c654f0f98861aae0f6f0602 2024-12-03T05:00:31,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6f4d19d52bc948f1aa40f3408f0773a5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6f4d19d52bc948f1aa40f3408f0773a5 2024-12-03T05:00:31,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/4c83ab82746641f78159edd42e0fd4f1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/4c83ab82746641f78159edd42e0fd4f1 2024-12-03T05:00:31,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a70e5f3c436548dfaa0ead0eabb701b8 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a70e5f3c436548dfaa0ead0eabb701b8 2024-12-03T05:00:31,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e71ca8a682074deebbd23ec36ee2fe5b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e71ca8a682074deebbd23ec36ee2fe5b 2024-12-03T05:00:31,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ecbf73f84e1e4a029bdb63e9af455736 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ecbf73f84e1e4a029bdb63e9af455736 2024-12-03T05:00:31,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6c3f36ec4e9a40a4ab245c6a63c2931d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/6c3f36ec4e9a40a4ab245c6a63c2931d 2024-12-03T05:00:31,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8774b27be3eb47d8b6f408e664a26ad0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8774b27be3eb47d8b6f408e664a26ad0 2024-12-03T05:00:31,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8f62f5c9cb6f4e5eb3964e7b99400688 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8f62f5c9cb6f4e5eb3964e7b99400688 2024-12-03T05:00:31,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ddf4df2dd1d541ed827029b6760ff07b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/ddf4df2dd1d541ed827029b6760ff07b 2024-12-03T05:00:31,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/bad8198fcb1045c7aa42f9bda5163be4 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/bad8198fcb1045c7aa42f9bda5163be4 2024-12-03T05:00:31,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a51c0745e5a144fe9ec562bfa6663c45 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a51c0745e5a144fe9ec562bfa6663c45 2024-12-03T05:00:31,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/93d45f622da44eeea159a7eebc1c5b9a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/93d45f622da44eeea159a7eebc1c5b9a 2024-12-03T05:00:31,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/864b9fc3e77c4b4e95d29ef538a3b164 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/864b9fc3e77c4b4e95d29ef538a3b164 2024-12-03T05:00:31,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a05084089a324a778eccf052481decfa to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a05084089a324a778eccf052481decfa 2024-12-03T05:00:31,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/d4ea5719deb84b51905b509c32ed7c53 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/d4ea5719deb84b51905b509c32ed7c53 2024-12-03T05:00:31,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e7214ed9e4eb4099ac2d1ad9005b2f1d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e7214ed9e4eb4099ac2d1ad9005b2f1d 2024-12-03T05:00:31,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/9d0b9e267919426b9ba3f8e6cfea85b6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/9d0b9e267919426b9ba3f8e6cfea85b6 2024-12-03T05:00:31,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/2ec25b25be424b7ea3893cbcb2816ad7 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/2ec25b25be424b7ea3893cbcb2816ad7 2024-12-03T05:00:31,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/593ecd9232354409bf3993faabf2180b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/593ecd9232354409bf3993faabf2180b 2024-12-03T05:00:31,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/c4327a67ab4845b0b3054258744b7d32 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/c4327a67ab4845b0b3054258744b7d32 2024-12-03T05:00:31,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a11f79b01c504c04af24ca6f019e8945 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/a11f79b01c504c04af24ca6f019e8945 2024-12-03T05:00:31,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/783737a244ab4f109bdce4d6d9fa7816 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/783737a244ab4f109bdce4d6d9fa7816 2024-12-03T05:00:31,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8acbfa260cac4597a7d5d16a399bc491 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/8acbfa260cac4597a7d5d16a399bc491 2024-12-03T05:00:31,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/81a38247ea6a4d388c24a25b0cb414dc to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/81a38247ea6a4d388c24a25b0cb414dc 2024-12-03T05:00:31,142 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/recovered.edits/428.seqid, newMaxSeqId=428, maxSeqId=4 2024-12-03T05:00:31,143 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. 2024-12-03T05:00:31,143 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for b4ec74c11861541ec3b2d9523601012e: Waiting for close lock at 1733202028929Running coprocessor pre-close hooks at 1733202028929Disabling compacts and flushes for region at 1733202028929Disabling writes for close at 1733202029359 (+430 ms)Obtaining lock to block concurrent updates at 1733202029359Preparing flush snapshotting stores in b4ec74c11861541ec3b2d9523601012e at 1733202029359Finished memstore snapshotting TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1733202029359Flushing stores of TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e. at 1733202029359Flushing b4ec74c11861541ec3b2d9523601012e/A: creating writer at 1733202029360 (+1 ms)Flushing mob file b4ec74c11861541ec3b2d9523601012e/A: appending metadata at 1733202029365 (+5 ms)Flushing mob file b4ec74c11861541ec3b2d9523601012e/A: closing flushed file at 1733202029365Flushing b4ec74c11861541ec3b2d9523601012e/A: appending metadata at 1733202029776 (+411 ms)Flushing b4ec74c11861541ec3b2d9523601012e/A: closing flushed file at 1733202029776Flushing b4ec74c11861541ec3b2d9523601012e/B: creating writer at 1733202030185 (+409 ms)Flushing b4ec74c11861541ec3b2d9523601012e/B: appending metadata at 1733202030188 (+3 ms)Flushing b4ec74c11861541ec3b2d9523601012e/B: closing flushed file at 1733202030188Flushing b4ec74c11861541ec3b2d9523601012e/C: creating writer at 1733202030599 (+411 ms)Flushing b4ec74c11861541ec3b2d9523601012e/C: appending metadata at 1733202030601 (+2 ms)Flushing b4ec74c11861541ec3b2d9523601012e/C: closing flushed file at 1733202030602 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55fd2190: reopening flushed file at 1733202031010 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@185a9d02: reopening flushed file at 1733202031015 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2924d02f: reopening flushed file at 1733202031019 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for b4ec74c11861541ec3b2d9523601012e in 1665ms, sequenceid=425, compaction requested=false at 1733202031025 (+6 ms)Writing region close event to WAL at 1733202031138 (+113 ms)Running coprocessor post-close hooks at 1733202031143 (+5 ms)Closed at 1733202031143 2024-12-03T05:00:31,146 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:31,146 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=b4ec74c11861541ec3b2d9523601012e, regionState=CLOSED 2024-12-03T05:00:31,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:00:31,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-03T05:00:31,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure b4ec74c11861541ec3b2d9523601012e, server=32beccd76b35,34719,1733201977102 in 2.3720 sec 2024-12-03T05:00:31,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-12-03T05:00:31,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4ec74c11861541ec3b2d9523601012e, UNASSIGN in 2.3780 sec 2024-12-03T05:00:31,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-03T05:00:31,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3820 sec 2024-12-03T05:00:31,156 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202031155"}]},"ts":"1733202031155"} 2024-12-03T05:00:31,157 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T05:00:31,157 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T05:00:31,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3930 sec 2024-12-03T05:00:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:00:32,921 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:32,921 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T05:00:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:32,924 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T05:00:32,925 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:32,928 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,931 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/recovered.edits] 2024-12-03T05:00:32,934 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/907657dfc55c46089c4640617256032d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/907657dfc55c46089c4640617256032d 2024-12-03T05:00:32,935 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/fa57ad0806304176931880d3a89ca0d3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/A/fa57ad0806304176931880d3a89ca0d3 2024-12-03T05:00:32,937 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/7310f07715e84d74a51b6eb7d9200c95 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/7310f07715e84d74a51b6eb7d9200c95 2024-12-03T05:00:32,939 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/f4c066e2d6c34dc3adf83c9e684133a9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/B/f4c066e2d6c34dc3adf83c9e684133a9 2024-12-03T05:00:32,941 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/c12b82ca531a4bc08bce5747d7e74513 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/c12b82ca531a4bc08bce5747d7e74513 2024-12-03T05:00:32,943 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e64204c7e0f844b991e511116b6019dd to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/C/e64204c7e0f844b991e511116b6019dd 2024-12-03T05:00:32,946 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/recovered.edits/428.seqid to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e/recovered.edits/428.seqid 2024-12-03T05:00:32,946 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,947 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T05:00:32,947 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T05:00:32,948 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T05:00:32,954 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203075c58c518234d01914e266da6ab2252_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203075c58c518234d01914e266da6ab2252_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,955 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031d88381f786b4add8953942af67c4a9d_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031d88381f786b4add8953942af67c4a9d_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,956 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032874359fb0e043508e6449cb60887c2a_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032874359fb0e043508e6449cb60887c2a_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,958 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120335249c7270d244d28908e0bbf3b80583_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120335249c7270d244d28908e0bbf3b80583_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,971 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203420e30552ca34e608f7f665bee7edd23_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203420e30552ca34e608f7f665bee7edd23_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,973 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034376dd4242904b1d8d439ea8f728d6ca_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034376dd4242904b1d8d439ea8f728d6ca_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,974 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034398c43201c244ce8b6670fea5d32d3c_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034398c43201c244ce8b6670fea5d32d3c_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,976 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035fdc38e6b7784f97a5c65a80e8f6b6de_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035fdc38e6b7784f97a5c65a80e8f6b6de_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,978 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036d6c9d6d268042aba0fb2a634b4215c5_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036d6c9d6d268042aba0fb2a634b4215c5_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,980 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120378b9c6242de344c59d794a2c37573e4a_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120378b9c6242de344c59d794a2c37573e4a_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,981 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038f1e0ba558fa42d4994c1fd7c4b75ca6_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038f1e0ba558fa42d4994c1fd7c4b75ca6_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,983 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203983d5b5f28fd4b19891ac81beacbae0f_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203983d5b5f28fd4b19891ac81beacbae0f_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,985 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a8e7913fdfd94126b4dd4917206d1588_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a8e7913fdfd94126b4dd4917206d1588_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,987 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b104bf23847942b88e117fa03ef2ae4e_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b104bf23847942b88e117fa03ef2ae4e_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,989 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bd4f8aec8aef4a8c9d5dc1bb31708d2f_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bd4f8aec8aef4a8c9d5dc1bb31708d2f_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,991 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c948c76dc7e14d48b4100601d068b7d6_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c948c76dc7e14d48b4100601d068b7d6_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,993 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cceebd1e34e5483e8c21337cfb7fd7b8_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cceebd1e34e5483e8c21337cfb7fd7b8_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,995 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d61462327f0d40df98f362fc0459c592_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d61462327f0d40df98f362fc0459c592_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,996 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dcae453076f442feb9a7ece38ea9f75d_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dcae453076f442feb9a7ece38ea9f75d_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,997 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e1d6850ee5a4421593d2ace9feaf1843_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e1d6850ee5a4421593d2ace9feaf1843_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:32,999 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ece775b718254a2d9585a91735dc48ed_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ece775b718254a2d9585a91735dc48ed_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:33,002 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ed0fabd3159f45f286021aa91eaea8e9_b4ec74c11861541ec3b2d9523601012e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ed0fabd3159f45f286021aa91eaea8e9_b4ec74c11861541ec3b2d9523601012e 2024-12-03T05:00:33,002 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T05:00:33,005 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:33,027 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T05:00:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T05:00:33,031 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T05:00:33,032 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:33,032 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T05:00:33,032 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733202033032"}]},"ts":"9223372036854775807"} 2024-12-03T05:00:33,034 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T05:00:33,034 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b4ec74c11861541ec3b2d9523601012e, NAME => 'TestAcidGuarantees,,1733202005454.b4ec74c11861541ec3b2d9523601012e.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T05:00:33,034 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T05:00:33,035 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733202033035"}]},"ts":"9223372036854775807"} 2024-12-03T05:00:33,036 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T05:00:33,037 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:33,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 116 msec 2024-12-03T05:00:33,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T05:00:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T05:00:33,244 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:33,254 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=247 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1161058445_22 at /127.0.0.1:54090 [Waiting for operation #383] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-775871842_22 at /127.0.0.1:54130 [Waiting for operation #313] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1161058445_22 at /127.0.0.1:38000 [Waiting for operation #856] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-775871842_22 at /127.0.0.1:60774 [Waiting for operation #823] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=911 (was 912), ProcessCount=11 (was 11), AvailableMemoryMB=4265 (was 3518) - AvailableMemoryMB LEAK? - 2024-12-03T05:00:33,266 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=247, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=911, ProcessCount=11, AvailableMemoryMB=4264 2024-12-03T05:00:33,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:00:33,268 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:00:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:33,271 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:00:33,271 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:33,271 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-12-03T05:00:33,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-03T05:00:33,272 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:00:33,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742072_1248 (size=959) 2024-12-03T05:00:33,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-03T05:00:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-03T05:00:33,687 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d 2024-12-03T05:00:33,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742073_1249 (size=53) 2024-12-03T05:00:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-03T05:00:34,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:00:34,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9ba85df8e95baa2667d284a9d65f61b1, disabling compactions & flushes 2024-12-03T05:00:34,097 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:34,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:34,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. after waiting 0 ms 2024-12-03T05:00:34,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:34,097 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:34,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9ba85df8e95baa2667d284a9d65f61b1: Waiting for close lock at 1733202034097Disabling compacts and flushes for region at 1733202034097Disabling writes for close at 1733202034097Writing region close event to WAL at 1733202034097Closed at 1733202034097 2024-12-03T05:00:34,098 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:00:34,099 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733202034098"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733202034098"}]},"ts":"1733202034098"} 2024-12-03T05:00:34,100 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:00:34,101 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:00:34,101 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202034101"}]},"ts":"1733202034101"} 2024-12-03T05:00:34,102 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T05:00:34,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9ba85df8e95baa2667d284a9d65f61b1, ASSIGN}] 2024-12-03T05:00:34,104 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9ba85df8e95baa2667d284a9d65f61b1, ASSIGN 2024-12-03T05:00:34,104 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9ba85df8e95baa2667d284a9d65f61b1, ASSIGN; state=OFFLINE, location=32beccd76b35,34719,1733201977102; forceNewPlan=false, retain=false 2024-12-03T05:00:34,255 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=9ba85df8e95baa2667d284a9d65f61b1, regionState=OPENING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:00:34,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9ba85df8e95baa2667d284a9d65f61b1, ASSIGN because future has completed 2024-12-03T05:00:34,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:00:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-03T05:00:34,412 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:34,412 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:00:34,412 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,413 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:00:34,413 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,413 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,414 INFO [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,415 INFO [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ba85df8e95baa2667d284a9d65f61b1 columnFamilyName A 2024-12-03T05:00:34,415 DEBUG [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:34,416 INFO [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] regionserver.HStore(327): Store=9ba85df8e95baa2667d284a9d65f61b1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:00:34,416 INFO [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,417 INFO [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ba85df8e95baa2667d284a9d65f61b1 columnFamilyName B 2024-12-03T05:00:34,417 DEBUG [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:34,417 INFO [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] regionserver.HStore(327): Store=9ba85df8e95baa2667d284a9d65f61b1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:00:34,418 INFO [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,419 INFO [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ba85df8e95baa2667d284a9d65f61b1 columnFamilyName C 2024-12-03T05:00:34,419 DEBUG [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:34,419 INFO [StoreOpener-9ba85df8e95baa2667d284a9d65f61b1-1 {}] regionserver.HStore(327): Store=9ba85df8e95baa2667d284a9d65f61b1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:00:34,419 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,420 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,420 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,421 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,421 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,422 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:00:34,423 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,426 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:00:34,426 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 9ba85df8e95baa2667d284a9d65f61b1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67723188, jitterRate=0.009154140949249268}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:00:34,426 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:34,427 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 9ba85df8e95baa2667d284a9d65f61b1: Running coprocessor pre-open hook at 1733202034413Writing region info on filesystem at 1733202034413Initializing all the Stores at 1733202034414 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202034414Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202034414Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202034414Cleaning up temporary data from old regions at 1733202034421 (+7 ms)Running coprocessor post-open hooks at 1733202034426 (+5 ms)Region opened successfully at 1733202034427 (+1 ms) 2024-12-03T05:00:34,427 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., pid=59, masterSystemTime=1733202034408 2024-12-03T05:00:34,429 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:34,429 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:34,430 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=9ba85df8e95baa2667d284a9d65f61b1, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:00:34,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:00:34,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-03T05:00:34,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 in 176 msec 2024-12-03T05:00:34,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-03T05:00:34,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9ba85df8e95baa2667d284a9d65f61b1, ASSIGN in 330 msec 2024-12-03T05:00:34,435 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:00:34,436 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202034435"}]},"ts":"1733202034435"} 2024-12-03T05:00:34,437 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T05:00:34,438 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:00:34,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-12-03T05:00:34,931 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:00:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-03T05:00:35,421 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:35,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:35,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:35,423 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542bcd2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:35,425 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e108666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,425 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:35,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:35,428 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:00:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,43423,1733201976038 2024-12-03T05:00:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c776660 2024-12-03T05:00:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:00:35,430 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:00:35,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:35,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:35,432 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ea88cc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:35,434 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:35,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9fb7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:35,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:35,438 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64dc14af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:35,439 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e41d7eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:35,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:35,442 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20efc0c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:35,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:35,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cd2ce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:35,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:35,445 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9ffd35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:35,446 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:35,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8bf347, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:35,447 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:35,448 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:35,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:35,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:35,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c9e984b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:35,449 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bbf051, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:35,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:35,451 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4991348b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:35,452 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf1196a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:35,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:35,454 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586bb521, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:35,455 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363f3749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:35,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:35,457 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b84d20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:35,458 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa07b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:35,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:35,460 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28cb4ba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:35,461 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28808c1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:00:35,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:00:35,463 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:00:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:00:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:00:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7f931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:00:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:00:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:35,464 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:00:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69da53c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:00:35,465 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:35,465 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:35,465 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:35,465 DEBUG [Thread-1155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:35,465 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:35,466 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:35,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:35,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:35,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:35,468 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:35,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:35,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:35,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:35,469 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:35,469 DEBUG [Thread-1153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-03T05:00:35,470 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:35,470 DEBUG [Thread-1159 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:35,471 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:35,471 DEBUG [Thread-1159 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:35,472 DEBUG [Thread-1164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:00:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T05:00:35,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:35,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:35,472 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:35,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:35,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:00:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:00:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:00:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:00:35,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:00:35,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:00:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:00:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:00:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:00:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:00:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:00:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:00:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:00:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:35,488 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:35,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:00:35,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/78c70e371fa0484ba2ab107decace1b9 is 50, key is test_row_0/A:col10/1733202035485/Put/seqid=0 2024-12-03T05:00:35,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202095527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202095529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202095529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742074_1250 (size=12001) 2024-12-03T05:00:35,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202095531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202095534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T05:00:35,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T05:00:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:35,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:35,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:35,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202095641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202095641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202095641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202095641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202095650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T05:00:35,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:35,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:35,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:35,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:35,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T05:00:35,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202095850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202095851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202095853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202095853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202095861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:35,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T05:00:35,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:35,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:35,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:35,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:35,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:35,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/78c70e371fa0484ba2ab107decace1b9 2024-12-03T05:00:35,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/fd2bcbfd57884d2e9a10b2fa7aa6de59 is 50, key is test_row_0/B:col10/1733202035485/Put/seqid=0 2024-12-03T05:00:35,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742075_1251 (size=12001) 2024-12-03T05:00:35,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/fd2bcbfd57884d2e9a10b2fa7aa6de59 2024-12-03T05:00:36,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/a4fce78bfe644c489bd15efeaea7f9e2 is 50, key is test_row_0/C:col10/1733202035485/Put/seqid=0 2024-12-03T05:00:36,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742076_1252 (size=12001) 2024-12-03T05:00:36,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T05:00:36,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:36,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:36,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:36,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:36,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T05:00:36,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202096161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202096161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202096161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202096162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202096170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T05:00:36,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:36,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:36,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:36,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:36,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:36,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T05:00:36,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:36,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:36,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:36,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:36,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:36,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/a4fce78bfe644c489bd15efeaea7f9e2 2024-12-03T05:00:36,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/78c70e371fa0484ba2ab107decace1b9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/78c70e371fa0484ba2ab107decace1b9 2024-12-03T05:00:36,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/78c70e371fa0484ba2ab107decace1b9, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T05:00:36,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/fd2bcbfd57884d2e9a10b2fa7aa6de59 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/fd2bcbfd57884d2e9a10b2fa7aa6de59 2024-12-03T05:00:36,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/fd2bcbfd57884d2e9a10b2fa7aa6de59, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T05:00:36,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/a4fce78bfe644c489bd15efeaea7f9e2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/a4fce78bfe644c489bd15efeaea7f9e2 2024-12-03T05:00:36,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/a4fce78bfe644c489bd15efeaea7f9e2, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T05:00:36,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 9ba85df8e95baa2667d284a9d65f61b1 in 992ms, sequenceid=14, compaction requested=false 2024-12-03T05:00:36,480 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T05:00:36,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:36,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T05:00:36,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:36,548 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-03T05:00:36,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/003afb8521d9469ba87e4db4911f14c0 is 50, key is test_row_0/A:col10/1733202035529/Put/seqid=0 2024-12-03T05:00:36,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742077_1253 (size=12001) 2024-12-03T05:00:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T05:00:36,669 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:00:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:36,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202096692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202096693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202096698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202096699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202096700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202096810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202096811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202096811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202096811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202096811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:36,964 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/003afb8521d9469ba87e4db4911f14c0 2024-12-03T05:00:36,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/6a5d5a1d1810411796a79856c7832c46 is 50, key is test_row_0/B:col10/1733202035529/Put/seqid=0 2024-12-03T05:00:37,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742078_1254 (size=12001) 2024-12-03T05:00:37,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202097022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202097022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202097022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202097022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202097022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202097331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202097330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202097331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202097331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202097331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,418 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/6a5d5a1d1810411796a79856c7832c46 2024-12-03T05:00:37,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/22830f634e674abd93eb9b6ae9eb6f6b is 50, key is test_row_0/C:col10/1733202035529/Put/seqid=0 2024-12-03T05:00:37,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742079_1255 (size=12001) 2024-12-03T05:00:37,457 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/22830f634e674abd93eb9b6ae9eb6f6b 2024-12-03T05:00:37,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/003afb8521d9469ba87e4db4911f14c0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/003afb8521d9469ba87e4db4911f14c0 2024-12-03T05:00:37,474 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/003afb8521d9469ba87e4db4911f14c0, entries=150, sequenceid=37, filesize=11.7 K 2024-12-03T05:00:37,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/6a5d5a1d1810411796a79856c7832c46 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/6a5d5a1d1810411796a79856c7832c46 2024-12-03T05:00:37,481 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/6a5d5a1d1810411796a79856c7832c46, entries=150, sequenceid=37, filesize=11.7 K 2024-12-03T05:00:37,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/22830f634e674abd93eb9b6ae9eb6f6b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/22830f634e674abd93eb9b6ae9eb6f6b 2024-12-03T05:00:37,488 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/22830f634e674abd93eb9b6ae9eb6f6b, entries=150, sequenceid=37, filesize=11.7 K 2024-12-03T05:00:37,489 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9ba85df8e95baa2667d284a9d65f61b1 in 941ms, sequenceid=37, compaction requested=false 2024-12-03T05:00:37,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:37,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:37,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-03T05:00:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-03T05:00:37,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-03T05:00:37,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0190 sec 2024-12-03T05:00:37,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.0270 sec 2024-12-03T05:00:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T05:00:37,620 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:37,621 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-03T05:00:37,623 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:00:37,624 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:37,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:00:37,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T05:00:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:37,777 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-03T05:00:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/6977c3bcbd94450b8ddc604befb7dba3 is 50, key is test_row_0/A:col10/1733202036693/Put/seqid=0 2024-12-03T05:00:37,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742080_1256 (size=12001) 2024-12-03T05:00:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:37,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:37,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202097870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202097872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202097873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202097875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202097876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:00:37,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202097980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202097981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202097981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202097991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202097991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202098190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202098190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:38,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202098190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:38,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202098201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:38,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202098201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:38,213 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/6977c3bcbd94450b8ddc604befb7dba3 2024-12-03T05:00:38,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/a326c7b88034471e82a8f3b7251cada7 is 50, key is test_row_0/B:col10/1733202036693/Put/seqid=0 2024-12-03T05:00:38,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742081_1257 (size=12001) 2024-12-03T05:00:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:00:38,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202098501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:38,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202098501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202098501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:38,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202098510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:38,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202098510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:38,634 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/a326c7b88034471e82a8f3b7251cada7 2024-12-03T05:00:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/d29b341fa06c4470be7143c9855a0caa is 50, key is test_row_0/C:col10/1733202036693/Put/seqid=0 2024-12-03T05:00:38,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742082_1258 (size=12001) 2024-12-03T05:00:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:00:39,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202099011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:39,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202099011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:39,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202099011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:39,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202099020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:39,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202099021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:39,087 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/d29b341fa06c4470be7143c9855a0caa 2024-12-03T05:00:39,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/6977c3bcbd94450b8ddc604befb7dba3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6977c3bcbd94450b8ddc604befb7dba3 2024-12-03T05:00:39,114 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6977c3bcbd94450b8ddc604befb7dba3, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T05:00:39,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/a326c7b88034471e82a8f3b7251cada7 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a326c7b88034471e82a8f3b7251cada7 2024-12-03T05:00:39,122 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a326c7b88034471e82a8f3b7251cada7, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T05:00:39,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/d29b341fa06c4470be7143c9855a0caa as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d29b341fa06c4470be7143c9855a0caa 2024-12-03T05:00:39,130 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d29b341fa06c4470be7143c9855a0caa, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T05:00:39,132 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 9ba85df8e95baa2667d284a9d65f61b1 in 1355ms, sequenceid=50, compaction requested=true 2024-12-03T05:00:39,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:39,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:39,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-03T05:00:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-03T05:00:39,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-03T05:00:39,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5150 sec 2024-12-03T05:00:39,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.5330 sec 2024-12-03T05:00:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:00:39,771 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:39,773 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-03T05:00:39,775 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T05:00:39,776 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:39,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T05:00:39,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T05:00:39,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:39,934 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-03T05:00:39,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/33bb65f8178e410ab19033d622adde57 is 50, key is test_row_0/A:col10/1733202037871/Put/seqid=0 2024-12-03T05:00:39,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742083_1259 (size=12001) 2024-12-03T05:00:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:40,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:40,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202100045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202100048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202100050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202100050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202100051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T05:00:40,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202100161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202100161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202100161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202100161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202100162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,371 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/33bb65f8178e410ab19033d622adde57 2024-12-03T05:00:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202100370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202100371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202100371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202100371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202100381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/ed99b6ad8750434d898267a6c21ecbf6 is 50, key is test_row_0/B:col10/1733202037871/Put/seqid=0 2024-12-03T05:00:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T05:00:40,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742084_1260 (size=12001) 2024-12-03T05:00:40,449 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/ed99b6ad8750434d898267a6c21ecbf6 2024-12-03T05:00:40,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/27c3bd63242d486fb15c5209f9ec7930 is 50, key is test_row_0/C:col10/1733202037871/Put/seqid=0 2024-12-03T05:00:40,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742085_1261 (size=12001) 2024-12-03T05:00:40,520 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/27c3bd63242d486fb15c5209f9ec7930 2024-12-03T05:00:40,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/33bb65f8178e410ab19033d622adde57 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/33bb65f8178e410ab19033d622adde57 2024-12-03T05:00:40,535 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/33bb65f8178e410ab19033d622adde57, entries=150, sequenceid=73, filesize=11.7 K 2024-12-03T05:00:40,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/ed99b6ad8750434d898267a6c21ecbf6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/ed99b6ad8750434d898267a6c21ecbf6 2024-12-03T05:00:40,542 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/ed99b6ad8750434d898267a6c21ecbf6, entries=150, sequenceid=73, filesize=11.7 K 2024-12-03T05:00:40,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/27c3bd63242d486fb15c5209f9ec7930 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/27c3bd63242d486fb15c5209f9ec7930 2024-12-03T05:00:40,549 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/27c3bd63242d486fb15c5209f9ec7930, entries=150, sequenceid=73, filesize=11.7 K 2024-12-03T05:00:40,557 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 9ba85df8e95baa2667d284a9d65f61b1 in 623ms, sequenceid=73, compaction requested=true 2024-12-03T05:00:40,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:40,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:40,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-03T05:00:40,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-03T05:00:40,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-03T05:00:40,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 783 msec 2024-12-03T05:00:40,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 794 msec 2024-12-03T05:00:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:40,701 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:40,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-03T05:00:40,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/2daf93f4a2df4a64af2cf218f8e3be2b is 50, key is test_row_0/A:col10/1733202040700/Put/seqid=0 2024-12-03T05:00:40,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742086_1262 (size=12001) 2024-12-03T05:00:40,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/2daf93f4a2df4a64af2cf218f8e3be2b 2024-12-03T05:00:40,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/0ef0033d9db940389ee65ae487885dd8 is 50, key is test_row_0/B:col10/1733202040700/Put/seqid=0 2024-12-03T05:00:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202100760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202100761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202100762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202100762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202100767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742087_1263 (size=12001) 2024-12-03T05:00:40,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202100880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202100881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202100881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202100881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202100881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T05:00:40,922 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:40,923 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-03T05:00:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T05:00:40,926 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:40,927 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:40,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T05:00:41,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T05:00:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202101091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202101093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202101093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202101094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202101101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/0ef0033d9db940389ee65ae487885dd8 2024-12-03T05:00:41,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/17cedcf19cc349f49642afa543797b39 is 50, key is test_row_0/C:col10/1733202040700/Put/seqid=0 2024-12-03T05:00:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T05:00:41,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T05:00:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742088_1264 (size=12001) 2024-12-03T05:00:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202101402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202101405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202101414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T05:00:41,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:41,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202101421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202101421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T05:00:41,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T05:00:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:41,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/17cedcf19cc349f49642afa543797b39 2024-12-03T05:00:41,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/2daf93f4a2df4a64af2cf218f8e3be2b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2daf93f4a2df4a64af2cf218f8e3be2b 2024-12-03T05:00:41,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2daf93f4a2df4a64af2cf218f8e3be2b, entries=150, sequenceid=91, filesize=11.7 K 2024-12-03T05:00:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/0ef0033d9db940389ee65ae487885dd8 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/0ef0033d9db940389ee65ae487885dd8 2024-12-03T05:00:41,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/0ef0033d9db940389ee65ae487885dd8, entries=150, sequenceid=91, filesize=11.7 K 2024-12-03T05:00:41,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/17cedcf19cc349f49642afa543797b39 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/17cedcf19cc349f49642afa543797b39 2024-12-03T05:00:41,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/17cedcf19cc349f49642afa543797b39, entries=150, sequenceid=91, filesize=11.7 K 2024-12-03T05:00:41,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 9ba85df8e95baa2667d284a9d65f61b1 in 1012ms, sequenceid=91, compaction requested=true 2024-12-03T05:00:41,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:41,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:41,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:41,714 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:00:41,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:41,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:41,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:41,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:41,714 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:00:41,717 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:00:41,717 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/B is initiating minor compaction (all files) 2024-12-03T05:00:41,717 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/B in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,717 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/fd2bcbfd57884d2e9a10b2fa7aa6de59, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/6a5d5a1d1810411796a79856c7832c46, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a326c7b88034471e82a8f3b7251cada7, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/ed99b6ad8750434d898267a6c21ecbf6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/0ef0033d9db940389ee65ae487885dd8] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=58.6 K 2024-12-03T05:00:41,717 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:00:41,717 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/A is initiating minor compaction (all files) 2024-12-03T05:00:41,717 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/A in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,718 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/78c70e371fa0484ba2ab107decace1b9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/003afb8521d9469ba87e4db4911f14c0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6977c3bcbd94450b8ddc604befb7dba3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/33bb65f8178e410ab19033d622adde57, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2daf93f4a2df4a64af2cf218f8e3be2b] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=58.6 K 2024-12-03T05:00:41,720 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting fd2bcbfd57884d2e9a10b2fa7aa6de59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733202035485 2024-12-03T05:00:41,720 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78c70e371fa0484ba2ab107decace1b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733202035485 2024-12-03T05:00:41,721 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 003afb8521d9469ba87e4db4911f14c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733202035524 2024-12-03T05:00:41,721 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a5d5a1d1810411796a79856c7832c46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733202035524 2024-12-03T05:00:41,721 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6977c3bcbd94450b8ddc604befb7dba3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733202036691 2024-12-03T05:00:41,721 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a326c7b88034471e82a8f3b7251cada7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733202036691 2024-12-03T05:00:41,722 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33bb65f8178e410ab19033d622adde57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1733202037862 2024-12-03T05:00:41,722 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ed99b6ad8750434d898267a6c21ecbf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1733202037862 2024-12-03T05:00:41,722 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ef0033d9db940389ee65ae487885dd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733202040049 2024-12-03T05:00:41,722 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2daf93f4a2df4a64af2cf218f8e3be2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733202040049 2024-12-03T05:00:41,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T05:00:41,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,747 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-03T05:00:41,752 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#B#compaction#216 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:41,753 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/5e1578c233534e3a9712bd541985cd39 is 50, key is test_row_0/B:col10/1733202040700/Put/seqid=0 2024-12-03T05:00:41,756 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#A#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:41,757 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/e21ff7bac4e04af29a5327c5702d6e3c is 50, key is test_row_0/A:col10/1733202040700/Put/seqid=0 2024-12-03T05:00:41,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742089_1265 (size=12173) 2024-12-03T05:00:41,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/32adccfe5d5e48008f80867b10b2f4f9 is 50, key is test_row_0/A:col10/1733202040725/Put/seqid=0 2024-12-03T05:00:41,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742091_1267 (size=12001) 2024-12-03T05:00:41,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742090_1266 (size=12173) 2024-12-03T05:00:41,927 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/e21ff7bac4e04af29a5327c5702d6e3c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/e21ff7bac4e04af29a5327c5702d6e3c 2024-12-03T05:00:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:41,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:41,943 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/A of 9ba85df8e95baa2667d284a9d65f61b1 into e21ff7bac4e04af29a5327c5702d6e3c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:41,943 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:41,943 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/A, priority=11, startTime=1733202041713; duration=0sec 2024-12-03T05:00:41,943 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:41,943 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:A 2024-12-03T05:00:41,943 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:00:41,946 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:00:41,946 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/C is initiating minor compaction (all files) 2024-12-03T05:00:41,946 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/C in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:41,946 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/a4fce78bfe644c489bd15efeaea7f9e2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/22830f634e674abd93eb9b6ae9eb6f6b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d29b341fa06c4470be7143c9855a0caa, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/27c3bd63242d486fb15c5209f9ec7930, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/17cedcf19cc349f49642afa543797b39] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=58.6 K 2024-12-03T05:00:41,947 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4fce78bfe644c489bd15efeaea7f9e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733202035485 2024-12-03T05:00:41,947 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22830f634e674abd93eb9b6ae9eb6f6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733202035524 2024-12-03T05:00:41,948 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d29b341fa06c4470be7143c9855a0caa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733202036691 2024-12-03T05:00:41,948 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27c3bd63242d486fb15c5209f9ec7930, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1733202037862 2024-12-03T05:00:41,949 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17cedcf19cc349f49642afa543797b39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733202040049 2024-12-03T05:00:41,970 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#C#compaction#219 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:41,971 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/233f952e6c7f4429a93260f7dc6d8cff is 50, key is test_row_0/C:col10/1733202040700/Put/seqid=0 2024-12-03T05:00:41,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202101963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742092_1268 (size=12173) 2024-12-03T05:00:41,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202101967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202101977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202101977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:41,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202101977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:41,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T05:00:42,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202102081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202102090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202102090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202102090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202102090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,197 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/5e1578c233534e3a9712bd541985cd39 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5e1578c233534e3a9712bd541985cd39 2024-12-03T05:00:42,201 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/B of 9ba85df8e95baa2667d284a9d65f61b1 into 5e1578c233534e3a9712bd541985cd39(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:42,201 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:42,202 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/B, priority=11, startTime=1733202041714; duration=0sec 2024-12-03T05:00:42,202 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:42,202 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:B 2024-12-03T05:00:42,263 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/32adccfe5d5e48008f80867b10b2f4f9 2024-12-03T05:00:42,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/26eaae7ff9b541d4a780cc3f7a441370 is 50, key is test_row_0/B:col10/1733202040725/Put/seqid=0 2024-12-03T05:00:42,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742093_1269 (size=12001) 2024-12-03T05:00:42,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202102291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202102300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202102300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202102300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202102301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,386 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/233f952e6c7f4429a93260f7dc6d8cff as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/233f952e6c7f4429a93260f7dc6d8cff 2024-12-03T05:00:42,399 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/C of 9ba85df8e95baa2667d284a9d65f61b1 into 233f952e6c7f4429a93260f7dc6d8cff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:42,399 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:42,399 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/C, priority=11, startTime=1733202041714; duration=0sec 2024-12-03T05:00:42,399 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:42,399 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:C 2024-12-03T05:00:42,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202102605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202102620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202102620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202102621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202102620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:42,689 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/26eaae7ff9b541d4a780cc3f7a441370 2024-12-03T05:00:42,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/4e6a72349aa847f8943b71f6f6535d74 is 50, key is test_row_0/C:col10/1733202040725/Put/seqid=0 2024-12-03T05:00:42,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742094_1270 (size=12001) 2024-12-03T05:00:42,720 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/4e6a72349aa847f8943b71f6f6535d74 2024-12-03T05:00:42,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/32adccfe5d5e48008f80867b10b2f4f9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/32adccfe5d5e48008f80867b10b2f4f9 2024-12-03T05:00:42,754 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/32adccfe5d5e48008f80867b10b2f4f9, entries=150, sequenceid=109, filesize=11.7 K 2024-12-03T05:00:42,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/26eaae7ff9b541d4a780cc3f7a441370 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/26eaae7ff9b541d4a780cc3f7a441370 2024-12-03T05:00:42,766 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/26eaae7ff9b541d4a780cc3f7a441370, entries=150, sequenceid=109, filesize=11.7 K 2024-12-03T05:00:42,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/4e6a72349aa847f8943b71f6f6535d74 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/4e6a72349aa847f8943b71f6f6535d74 2024-12-03T05:00:42,794 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/4e6a72349aa847f8943b71f6f6535d74, entries=150, sequenceid=109, filesize=11.7 K 2024-12-03T05:00:42,796 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 9ba85df8e95baa2667d284a9d65f61b1 in 1049ms, sequenceid=109, compaction requested=false 2024-12-03T05:00:42,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:42,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:42,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-03T05:00:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-03T05:00:42,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-03T05:00:42,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8710 sec 2024-12-03T05:00:42,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.8840 sec 2024-12-03T05:00:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T05:00:43,071 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:43,071 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-03T05:00:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:00:43,076 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:43,077 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:43,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:43,123 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:43,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-03T05:00:43,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/3231ed4e11c14f70bb3f5958f938f24e is 50, key is test_row_0/A:col10/1733202041970/Put/seqid=0 2024-12-03T05:00:43,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742095_1271 (size=12051) 2024-12-03T05:00:43,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202103162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202103163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202103165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202103171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202103171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:00:43,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T05:00:43,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:43,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:43,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:43,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:43,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:43,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202103271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202103271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202103280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T05:00:43,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:43,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:43,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:43,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:43,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:00:43,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202103481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202103481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202103490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T05:00:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:43,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:43,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/3231ed4e11c14f70bb3f5958f938f24e 2024-12-03T05:00:43,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/1b973b3b05f049d6b48651a5467578d8 is 50, key is test_row_0/B:col10/1733202041970/Put/seqid=0 2024-12-03T05:00:43,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742096_1272 (size=12051) 2024-12-03T05:00:43,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/1b973b3b05f049d6b48651a5467578d8 2024-12-03T05:00:43,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/13561547f38544c0938d94af9d57494e is 50, key is test_row_0/C:col10/1733202041970/Put/seqid=0 2024-12-03T05:00:43,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742097_1273 (size=12051) 2024-12-03T05:00:43,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/13561547f38544c0938d94af9d57494e 2024-12-03T05:00:43,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/3231ed4e11c14f70bb3f5958f938f24e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3231ed4e11c14f70bb3f5958f938f24e 2024-12-03T05:00:43,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3231ed4e11c14f70bb3f5958f938f24e, entries=150, sequenceid=131, filesize=11.8 K 2024-12-03T05:00:43,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/1b973b3b05f049d6b48651a5467578d8 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/1b973b3b05f049d6b48651a5467578d8 2024-12-03T05:00:43,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/1b973b3b05f049d6b48651a5467578d8, entries=150, sequenceid=131, filesize=11.8 K 2024-12-03T05:00:43,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/13561547f38544c0938d94af9d57494e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/13561547f38544c0938d94af9d57494e 2024-12-03T05:00:43,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/13561547f38544c0938d94af9d57494e, entries=150, sequenceid=131, filesize=11.8 K 2024-12-03T05:00:43,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 9ba85df8e95baa2667d284a9d65f61b1 in 549ms, sequenceid=131, compaction requested=true 2024-12-03T05:00:43,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:43,673 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:43,674 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:43,674 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/A is initiating minor compaction (all files) 2024-12-03T05:00:43,674 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/A in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:43,674 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/e21ff7bac4e04af29a5327c5702d6e3c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/32adccfe5d5e48008f80867b10b2f4f9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3231ed4e11c14f70bb3f5958f938f24e] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=35.4 K 2024-12-03T05:00:43,675 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e21ff7bac4e04af29a5327c5702d6e3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733202040049 2024-12-03T05:00:43,676 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32adccfe5d5e48008f80867b10b2f4f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733202040725 2024-12-03T05:00:43,677 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3231ed4e11c14f70bb3f5958f938f24e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733202041966 2024-12-03T05:00:43,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:43,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:43,680 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:43,683 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:43,683 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/B is initiating minor compaction (all files) 2024-12-03T05:00:43,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:43,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:43,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:43,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:43,692 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/B in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:43,692 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5e1578c233534e3a9712bd541985cd39, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/26eaae7ff9b541d4a780cc3f7a441370, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/1b973b3b05f049d6b48651a5467578d8] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=35.4 K 2024-12-03T05:00:43,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T05:00:43,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:43,694 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e1578c233534e3a9712bd541985cd39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733202040049 2024-12-03T05:00:43,694 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 26eaae7ff9b541d4a780cc3f7a441370, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733202040725 2024-12-03T05:00:43,695 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b973b3b05f049d6b48651a5467578d8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733202041966 2024-12-03T05:00:43,697 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-03T05:00:43,699 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#A#compaction#225 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:43,699 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/5d789a235edd41c5b6268dc331a8bceb is 50, key is test_row_0/A:col10/1733202041970/Put/seqid=0 2024-12-03T05:00:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:00:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/37d367318b9f4070848edb3c3b7d6634 is 50, key is test_row_0/A:col10/1733202043163/Put/seqid=0 2024-12-03T05:00:43,719 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#B#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:43,720 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/5a27b19e132143fc8922cea4b2f3b15e is 50, key is test_row_0/B:col10/1733202041970/Put/seqid=0 2024-12-03T05:00:43,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742098_1274 (size=12325) 2024-12-03T05:00:43,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742099_1275 (size=12151) 2024-12-03T05:00:43,791 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/37d367318b9f4070848edb3c3b7d6634 2024-12-03T05:00:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:43,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:43,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742100_1276 (size=12325) 2024-12-03T05:00:43,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/dc2b26bcd9214e3ca0f3dff52c31f58f is 50, key is test_row_0/B:col10/1733202043163/Put/seqid=0 2024-12-03T05:00:43,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202103834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202103834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742101_1277 (size=12151) 2024-12-03T05:00:43,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,847 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/dc2b26bcd9214e3ca0f3dff52c31f58f 2024-12-03T05:00:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202103839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/b95c90223f7d4edbb3fe54a9b2c40122 is 50, key is test_row_0/C:col10/1733202043163/Put/seqid=0 2024-12-03T05:00:43,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742102_1278 (size=12151) 2024-12-03T05:00:43,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202103951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202103951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202103951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:43,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:43,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202104161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202104161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202104161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,182 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/5d789a235edd41c5b6268dc331a8bceb as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/5d789a235edd41c5b6268dc331a8bceb 2024-12-03T05:00:44,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202104181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,186 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in 9ba85df8e95baa2667d284a9d65f61b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,191 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/A of 9ba85df8e95baa2667d284a9d65f61b1 into 5d789a235edd41c5b6268dc331a8bceb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:44,191 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:44,191 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/A, priority=13, startTime=1733202043673; duration=0sec 2024-12-03T05:00:44,191 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:44,191 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:A 2024-12-03T05:00:44,191 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:44,193 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:44,194 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/C is initiating minor compaction (all files) 2024-12-03T05:00:44,194 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/C in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:44,194 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/233f952e6c7f4429a93260f7dc6d8cff, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/4e6a72349aa847f8943b71f6f6535d74, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/13561547f38544c0938d94af9d57494e] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=35.4 K 2024-12-03T05:00:44,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202104192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,195 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in 9ba85df8e95baa2667d284a9d65f61b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,195 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 233f952e6c7f4429a93260f7dc6d8cff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733202040049 2024-12-03T05:00:44,196 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e6a72349aa847f8943b71f6f6535d74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733202040725 2024-12-03T05:00:44,196 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13561547f38544c0938d94af9d57494e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733202041966 2024-12-03T05:00:44,206 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:44,206 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/8d90507f65804bc09f0fe7a4bba0bbed is 50, key is test_row_0/C:col10/1733202041970/Put/seqid=0 2024-12-03T05:00:44,212 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/5a27b19e132143fc8922cea4b2f3b15e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5a27b19e132143fc8922cea4b2f3b15e 2024-12-03T05:00:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:00:44,221 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/B of 9ba85df8e95baa2667d284a9d65f61b1 into 5a27b19e132143fc8922cea4b2f3b15e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:44,221 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:44,221 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/B, priority=13, startTime=1733202043680; duration=0sec 2024-12-03T05:00:44,222 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:44,222 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:B 2024-12-03T05:00:44,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742103_1279 (size=12325) 2024-12-03T05:00:44,237 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/8d90507f65804bc09f0fe7a4bba0bbed as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8d90507f65804bc09f0fe7a4bba0bbed 2024-12-03T05:00:44,251 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/C of 9ba85df8e95baa2667d284a9d65f61b1 into 8d90507f65804bc09f0fe7a4bba0bbed(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:44,251 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:44,251 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/C, priority=13, startTime=1733202043685; duration=0sec 2024-12-03T05:00:44,251 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:44,251 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:C 2024-12-03T05:00:44,299 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/b95c90223f7d4edbb3fe54a9b2c40122 2024-12-03T05:00:44,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/37d367318b9f4070848edb3c3b7d6634 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/37d367318b9f4070848edb3c3b7d6634 2024-12-03T05:00:44,322 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/37d367318b9f4070848edb3c3b7d6634, entries=150, sequenceid=148, filesize=11.9 K 2024-12-03T05:00:44,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/dc2b26bcd9214e3ca0f3dff52c31f58f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/dc2b26bcd9214e3ca0f3dff52c31f58f 2024-12-03T05:00:44,328 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/dc2b26bcd9214e3ca0f3dff52c31f58f, entries=150, sequenceid=148, filesize=11.9 K 2024-12-03T05:00:44,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/b95c90223f7d4edbb3fe54a9b2c40122 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b95c90223f7d4edbb3fe54a9b2c40122 2024-12-03T05:00:44,334 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b95c90223f7d4edbb3fe54a9b2c40122, entries=150, sequenceid=148, filesize=11.9 K 2024-12-03T05:00:44,336 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 9ba85df8e95baa2667d284a9d65f61b1 in 642ms, sequenceid=148, compaction requested=false 2024-12-03T05:00:44,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:44,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:44,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-03T05:00:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-03T05:00:44,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-03T05:00:44,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2650 sec 2024-12-03T05:00:44,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2740 sec 2024-12-03T05:00:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:44,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-03T05:00:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:44,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/98a2e913de1e4461b67cfdeb8ba44af1 is 50, key is test_row_0/A:col10/1733202044475/Put/seqid=0 2024-12-03T05:00:44,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202104498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202104499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202104502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742104_1280 (size=14541) 2024-12-03T05:00:44,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202104612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202104622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202104622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202104821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202104830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202104830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:44,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/98a2e913de1e4461b67cfdeb8ba44af1 2024-12-03T05:00:44,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/9d6e02a66751483e93cfdcfd2b1bffe3 is 50, key is test_row_0/B:col10/1733202044475/Put/seqid=0 2024-12-03T05:00:44,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742105_1281 (size=12151) 2024-12-03T05:00:44,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/9d6e02a66751483e93cfdcfd2b1bffe3 2024-12-03T05:00:44,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/25a63c7f643e4c2392d043232736d2c1 is 50, key is test_row_0/C:col10/1733202044475/Put/seqid=0 2024-12-03T05:00:45,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742106_1282 (size=12151) 2024-12-03T05:00:45,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202105131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:45,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:45,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202105140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:45,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202105140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:00:45,230 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:45,231 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-03T05:00:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T05:00:45,235 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:45,235 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:45,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T05:00:45,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T05:00:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:45,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:45,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/25a63c7f643e4c2392d043232736d2c1 2024-12-03T05:00:45,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/98a2e913de1e4461b67cfdeb8ba44af1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/98a2e913de1e4461b67cfdeb8ba44af1 2024-12-03T05:00:45,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/98a2e913de1e4461b67cfdeb8ba44af1, entries=200, sequenceid=172, filesize=14.2 K 2024-12-03T05:00:45,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/9d6e02a66751483e93cfdcfd2b1bffe3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/9d6e02a66751483e93cfdcfd2b1bffe3 2024-12-03T05:00:45,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/9d6e02a66751483e93cfdcfd2b1bffe3, entries=150, sequenceid=172, filesize=11.9 K 2024-12-03T05:00:45,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/25a63c7f643e4c2392d043232736d2c1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/25a63c7f643e4c2392d043232736d2c1 2024-12-03T05:00:45,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/25a63c7f643e4c2392d043232736d2c1, entries=150, sequenceid=172, filesize=11.9 K 2024-12-03T05:00:45,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 9ba85df8e95baa2667d284a9d65f61b1 in 995ms, sequenceid=172, compaction requested=true 2024-12-03T05:00:45,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:45,471 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:45,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:45,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:45,471 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:45,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:45,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:45,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:45,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:45,472 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39017 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:45,472 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/A is initiating minor compaction (all files) 2024-12-03T05:00:45,473 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/A in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:45,473 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/5d789a235edd41c5b6268dc331a8bceb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/37d367318b9f4070848edb3c3b7d6634, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/98a2e913de1e4461b67cfdeb8ba44af1] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=38.1 K 2024-12-03T05:00:45,473 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d789a235edd41c5b6268dc331a8bceb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733202041966 2024-12-03T05:00:45,473 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:45,474 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/B is initiating minor compaction (all files) 2024-12-03T05:00:45,474 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/B in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:45,474 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5a27b19e132143fc8922cea4b2f3b15e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/dc2b26bcd9214e3ca0f3dff52c31f58f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/9d6e02a66751483e93cfdcfd2b1bffe3] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=35.8 K 2024-12-03T05:00:45,475 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37d367318b9f4070848edb3c3b7d6634, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733202043154 2024-12-03T05:00:45,475 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a27b19e132143fc8922cea4b2f3b15e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733202041966 2024-12-03T05:00:45,476 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98a2e913de1e4461b67cfdeb8ba44af1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733202043800 2024-12-03T05:00:45,477 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting dc2b26bcd9214e3ca0f3dff52c31f58f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733202043154 2024-12-03T05:00:45,478 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d6e02a66751483e93cfdcfd2b1bffe3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733202043837 2024-12-03T05:00:45,503 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#A#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:45,504 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/57e4a954024c4ee99f69c526a0d20bf6 is 50, key is test_row_0/A:col10/1733202044475/Put/seqid=0 2024-12-03T05:00:45,515 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:45,516 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/d60c403c3b22481aa403b249918460d0 is 50, key is test_row_0/B:col10/1733202044475/Put/seqid=0 2024-12-03T05:00:45,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742107_1283 (size=12527) 2024-12-03T05:00:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T05:00:45,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T05:00:45,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:45,556 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-03T05:00:45,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742108_1284 (size=12527) 2024-12-03T05:00:45,571 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/57e4a954024c4ee99f69c526a0d20bf6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/57e4a954024c4ee99f69c526a0d20bf6 2024-12-03T05:00:45,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/65c51e3fb5e74c049deaf6676bf03da1 is 50, key is test_row_0/A:col10/1733202044498/Put/seqid=0 2024-12-03T05:00:45,594 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/A of 9ba85df8e95baa2667d284a9d65f61b1 into 57e4a954024c4ee99f69c526a0d20bf6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:45,594 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:45,594 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/A, priority=13, startTime=1733202045470; duration=0sec 2024-12-03T05:00:45,594 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:45,594 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:A 2024-12-03T05:00:45,594 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:45,597 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:45,598 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/C is initiating minor compaction (all files) 2024-12-03T05:00:45,598 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/C in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:45,598 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8d90507f65804bc09f0fe7a4bba0bbed, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b95c90223f7d4edbb3fe54a9b2c40122, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/25a63c7f643e4c2392d043232736d2c1] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=35.8 K 2024-12-03T05:00:45,600 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d90507f65804bc09f0fe7a4bba0bbed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733202041966 2024-12-03T05:00:45,600 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b95c90223f7d4edbb3fe54a9b2c40122, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733202043154 2024-12-03T05:00:45,601 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25a63c7f643e4c2392d043232736d2c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733202043837 2024-12-03T05:00:45,616 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/d60c403c3b22481aa403b249918460d0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/d60c403c3b22481aa403b249918460d0 2024-12-03T05:00:45,623 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#C#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:45,624 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/8b9d2e028084419299dad765cb38c8ae is 50, key is test_row_0/C:col10/1733202044475/Put/seqid=0 2024-12-03T05:00:45,645 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/B of 9ba85df8e95baa2667d284a9d65f61b1 into d60c403c3b22481aa403b249918460d0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:45,645 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:45,646 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/B, priority=13, startTime=1733202045471; duration=0sec 2024-12-03T05:00:45,646 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:45,646 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:B 2024-12-03T05:00:45,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742109_1285 (size=12151) 2024-12-03T05:00:45,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:45,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742110_1286 (size=12527) 2024-12-03T05:00:45,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202105760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:45,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202105760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:45,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202105760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T05:00:45,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202105874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:45,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:45,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202105874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202105874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,065 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/65c51e3fb5e74c049deaf6676bf03da1 2024-12-03T05:00:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202106081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202106081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:46,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202106081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/8e69274e76ba4237a35f523bcf19dda1 is 50, key is test_row_0/B:col10/1733202044498/Put/seqid=0 2024-12-03T05:00:46,114 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/8b9d2e028084419299dad765cb38c8ae as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8b9d2e028084419299dad765cb38c8ae 2024-12-03T05:00:46,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742111_1287 (size=12151) 2024-12-03T05:00:46,122 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/C of 9ba85df8e95baa2667d284a9d65f61b1 into 8b9d2e028084419299dad765cb38c8ae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:46,122 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:46,122 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/C, priority=13, startTime=1733202045472; duration=0sec 2024-12-03T05:00:46,123 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:46,123 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:C 2024-12-03T05:00:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202106206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:46,212 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in 9ba85df8e95baa2667d284a9d65f61b1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202106218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:46,221 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in 9ba85df8e95baa2667d284a9d65f61b1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:00:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T05:00:46,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202106390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:46,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202106391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:46,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202106391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,529 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/8e69274e76ba4237a35f523bcf19dda1 2024-12-03T05:00:46,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/d110033cd6e0498b92806c6bbbdae8d4 is 50, key is test_row_0/C:col10/1733202044498/Put/seqid=0 2024-12-03T05:00:46,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742112_1288 (size=12151) 2024-12-03T05:00:46,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202106900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:46,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202106900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202106910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:46,989 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/d110033cd6e0498b92806c6bbbdae8d4 2024-12-03T05:00:46,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/65c51e3fb5e74c049deaf6676bf03da1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/65c51e3fb5e74c049deaf6676bf03da1 2024-12-03T05:00:47,004 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/65c51e3fb5e74c049deaf6676bf03da1, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T05:00:47,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/8e69274e76ba4237a35f523bcf19dda1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/8e69274e76ba4237a35f523bcf19dda1 2024-12-03T05:00:47,008 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/8e69274e76ba4237a35f523bcf19dda1, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T05:00:47,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/d110033cd6e0498b92806c6bbbdae8d4 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d110033cd6e0498b92806c6bbbdae8d4 2024-12-03T05:00:47,014 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d110033cd6e0498b92806c6bbbdae8d4, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T05:00:47,015 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 9ba85df8e95baa2667d284a9d65f61b1 in 1459ms, sequenceid=187, compaction requested=false 2024-12-03T05:00:47,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:47,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:47,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-03T05:00:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-03T05:00:47,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-03T05:00:47,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7810 sec 2024-12-03T05:00:47,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7890 sec 2024-12-03T05:00:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T05:00:47,380 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:47,381 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-03T05:00:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:00:47,384 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:47,390 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:47,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:00:47,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T05:00:47,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:47,546 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-03T05:00:47,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/719b90e54fa6465083dddf12c22a4500 is 50, key is test_row_0/A:col10/1733202045712/Put/seqid=0 2024-12-03T05:00:47,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742113_1289 (size=12151) 2024-12-03T05:00:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:00:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:47,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202107941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:47,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202107942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:47,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202107946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:47,968 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/719b90e54fa6465083dddf12c22a4500 2024-12-03T05:00:47,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/7e318c9067cb439ca1913f3f37021a0f is 50, key is test_row_0/B:col10/1733202045712/Put/seqid=0 2024-12-03T05:00:47,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742114_1290 (size=12151) 2024-12-03T05:00:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:00:48,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202108050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:48,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202108050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:48,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202108060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:48,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202108260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:48,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202108260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:48,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202108270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:48,394 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/7e318c9067cb439ca1913f3f37021a0f 2024-12-03T05:00:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/1dafaf11226f4e4caa7f0b54bd98dd55 is 50, key is test_row_0/C:col10/1733202045712/Put/seqid=0 2024-12-03T05:00:48,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742115_1291 (size=12151) 2024-12-03T05:00:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:00:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202108571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202108580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:48,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202108580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:48,848 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/1dafaf11226f4e4caa7f0b54bd98dd55 2024-12-03T05:00:48,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/719b90e54fa6465083dddf12c22a4500 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/719b90e54fa6465083dddf12c22a4500 2024-12-03T05:00:48,857 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/719b90e54fa6465083dddf12c22a4500, entries=150, sequenceid=211, filesize=11.9 K 2024-12-03T05:00:48,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/7e318c9067cb439ca1913f3f37021a0f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/7e318c9067cb439ca1913f3f37021a0f 2024-12-03T05:00:48,862 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/7e318c9067cb439ca1913f3f37021a0f, entries=150, sequenceid=211, filesize=11.9 K 2024-12-03T05:00:48,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/1dafaf11226f4e4caa7f0b54bd98dd55 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/1dafaf11226f4e4caa7f0b54bd98dd55 2024-12-03T05:00:48,866 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/1dafaf11226f4e4caa7f0b54bd98dd55, entries=150, sequenceid=211, filesize=11.9 K 2024-12-03T05:00:48,867 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 9ba85df8e95baa2667d284a9d65f61b1 in 1321ms, sequenceid=211, compaction requested=true 2024-12-03T05:00:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:48,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-03T05:00:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-03T05:00:48,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-03T05:00:48,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-12-03T05:00:48,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4930 sec 2024-12-03T05:00:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:49,086 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:49,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-03T05:00:49,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/2d22d3059f50426ebb75a58eac5778d5 is 50, key is test_row_0/A:col10/1733202047923/Put/seqid=0 2024-12-03T05:00:49,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742116_1292 (size=14541) 2024-12-03T05:00:49,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/2d22d3059f50426ebb75a58eac5778d5 2024-12-03T05:00:49,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/3a2eda64270847698061e8e2442cde63 is 50, key is test_row_0/B:col10/1733202047923/Put/seqid=0 2024-12-03T05:00:49,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202109123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202109128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202109128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742117_1293 (size=12151) 2024-12-03T05:00:49,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/3a2eda64270847698061e8e2442cde63 2024-12-03T05:00:49,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/8c8a3ae858634a85b3594122017c6405 is 50, key is test_row_0/C:col10/1733202047923/Put/seqid=0 2024-12-03T05:00:49,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742118_1294 (size=12151) 2024-12-03T05:00:49,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202109240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202109240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202109240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202109450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202109451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202109451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:00:49,531 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:49,532 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-03T05:00:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T05:00:49,534 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:49,534 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:49,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:49,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/8c8a3ae858634a85b3594122017c6405 2024-12-03T05:00:49,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/2d22d3059f50426ebb75a58eac5778d5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2d22d3059f50426ebb75a58eac5778d5 2024-12-03T05:00:49,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2d22d3059f50426ebb75a58eac5778d5, entries=200, sequenceid=227, filesize=14.2 K 2024-12-03T05:00:49,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/3a2eda64270847698061e8e2442cde63 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/3a2eda64270847698061e8e2442cde63 2024-12-03T05:00:49,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/3a2eda64270847698061e8e2442cde63, entries=150, sequenceid=227, filesize=11.9 K 2024-12-03T05:00:49,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/8c8a3ae858634a85b3594122017c6405 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8c8a3ae858634a85b3594122017c6405 2024-12-03T05:00:49,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8c8a3ae858634a85b3594122017c6405, entries=150, sequenceid=227, filesize=11.9 K 2024-12-03T05:00:49,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 9ba85df8e95baa2667d284a9d65f61b1 in 508ms, sequenceid=227, compaction requested=true 2024-12-03T05:00:49,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:49,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:49,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:49,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:49,595 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:49,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:49,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:49,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:49,595 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:49,596 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:49,596 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51370 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:49,596 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/B is initiating minor compaction (all files) 2024-12-03T05:00:49,596 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/A is initiating minor compaction (all files) 2024-12-03T05:00:49,596 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/B in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:49,596 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/A in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:49,596 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/d60c403c3b22481aa403b249918460d0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/8e69274e76ba4237a35f523bcf19dda1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/7e318c9067cb439ca1913f3f37021a0f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/3a2eda64270847698061e8e2442cde63] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=47.8 K 2024-12-03T05:00:49,596 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/57e4a954024c4ee99f69c526a0d20bf6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/65c51e3fb5e74c049deaf6676bf03da1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/719b90e54fa6465083dddf12c22a4500, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2d22d3059f50426ebb75a58eac5778d5] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=50.2 K 2024-12-03T05:00:49,597 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting d60c403c3b22481aa403b249918460d0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733202043837 2024-12-03T05:00:49,597 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e69274e76ba4237a35f523bcf19dda1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733202044495 2024-12-03T05:00:49,597 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e4a954024c4ee99f69c526a0d20bf6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733202043837 2024-12-03T05:00:49,598 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e318c9067cb439ca1913f3f37021a0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733202045712 2024-12-03T05:00:49,598 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65c51e3fb5e74c049deaf6676bf03da1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733202044495 2024-12-03T05:00:49,599 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 719b90e54fa6465083dddf12c22a4500, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733202045712 2024-12-03T05:00:49,599 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a2eda64270847698061e8e2442cde63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733202047923 2024-12-03T05:00:49,600 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d22d3059f50426ebb75a58eac5778d5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733202047923 2024-12-03T05:00:49,611 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:49,611 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/16d10a74c0bf4ce7aeae0ae5da8c3817 is 50, key is test_row_0/A:col10/1733202047923/Put/seqid=0 2024-12-03T05:00:49,615 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#B#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:49,616 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/a74ab0dcdfa04f979ce5f565d25e36eb is 50, key is test_row_0/B:col10/1733202047923/Put/seqid=0 2024-12-03T05:00:49,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742119_1295 (size=12663) 2024-12-03T05:00:49,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742120_1296 (size=12663) 2024-12-03T05:00:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T05:00:49,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T05:00:49,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:49,687 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-03T05:00:49,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/f894cc958712457f8a1ea7d187855c06 is 50, key is test_row_0/A:col10/1733202049121/Put/seqid=0 2024-12-03T05:00:49,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742121_1297 (size=12151) 2024-12-03T05:00:49,697 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/f894cc958712457f8a1ea7d187855c06 2024-12-03T05:00:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/61f6f283e59a4bcdbdd6846bf8d2e043 is 50, key is test_row_0/B:col10/1733202049121/Put/seqid=0 2024-12-03T05:00:49,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742122_1298 (size=12151) 2024-12-03T05:00:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:49,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:49,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202109784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202109785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202109785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T05:00:49,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202109892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202109901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202109901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,054 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/16d10a74c0bf4ce7aeae0ae5da8c3817 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/16d10a74c0bf4ce7aeae0ae5da8c3817 2024-12-03T05:00:50,071 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/A of 9ba85df8e95baa2667d284a9d65f61b1 into 16d10a74c0bf4ce7aeae0ae5da8c3817(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:50,071 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:50,071 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/A, priority=12, startTime=1733202049594; duration=0sec 2024-12-03T05:00:50,072 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:50,072 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:A 2024-12-03T05:00:50,072 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:50,075 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/a74ab0dcdfa04f979ce5f565d25e36eb as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a74ab0dcdfa04f979ce5f565d25e36eb 2024-12-03T05:00:50,079 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:50,079 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/C is initiating minor compaction (all files) 2024-12-03T05:00:50,079 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/C in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:50,079 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8b9d2e028084419299dad765cb38c8ae, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d110033cd6e0498b92806c6bbbdae8d4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/1dafaf11226f4e4caa7f0b54bd98dd55, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8c8a3ae858634a85b3594122017c6405] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=47.8 K 2024-12-03T05:00:50,093 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b9d2e028084419299dad765cb38c8ae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733202043837 2024-12-03T05:00:50,101 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d110033cd6e0498b92806c6bbbdae8d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733202044495 2024-12-03T05:00:50,108 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dafaf11226f4e4caa7f0b54bd98dd55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733202045712 2024-12-03T05:00:50,113 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c8a3ae858634a85b3594122017c6405, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733202047923 2024-12-03T05:00:50,116 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/61f6f283e59a4bcdbdd6846bf8d2e043 2024-12-03T05:00:50,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202110111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202110111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202110111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,120 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/B of 9ba85df8e95baa2667d284a9d65f61b1 into a74ab0dcdfa04f979ce5f565d25e36eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:50,121 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:50,122 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/B, priority=12, startTime=1733202049594; duration=0sec 2024-12-03T05:00:50,122 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:50,122 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:B 2024-12-03T05:00:50,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/dc7b395a55914facaa8ffcd292cbbdcf is 50, key is test_row_0/C:col10/1733202049121/Put/seqid=0 2024-12-03T05:00:50,156 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:50,156 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/9722dd4b51e44ca99307736a149c5677 is 50, key is test_row_0/C:col10/1733202047923/Put/seqid=0 2024-12-03T05:00:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T05:00:50,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742123_1299 (size=12151) 2024-12-03T05:00:50,164 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/dc7b395a55914facaa8ffcd292cbbdcf 2024-12-03T05:00:50,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/f894cc958712457f8a1ea7d187855c06 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/f894cc958712457f8a1ea7d187855c06 2024-12-03T05:00:50,177 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/f894cc958712457f8a1ea7d187855c06, entries=150, sequenceid=247, filesize=11.9 K 2024-12-03T05:00:50,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/61f6f283e59a4bcdbdd6846bf8d2e043 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/61f6f283e59a4bcdbdd6846bf8d2e043 2024-12-03T05:00:50,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742124_1300 (size=12663) 2024-12-03T05:00:50,186 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/61f6f283e59a4bcdbdd6846bf8d2e043, entries=150, sequenceid=247, filesize=11.9 K 2024-12-03T05:00:50,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/dc7b395a55914facaa8ffcd292cbbdcf as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dc7b395a55914facaa8ffcd292cbbdcf 2024-12-03T05:00:50,192 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/9722dd4b51e44ca99307736a149c5677 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/9722dd4b51e44ca99307736a149c5677 2024-12-03T05:00:50,199 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/C of 9ba85df8e95baa2667d284a9d65f61b1 into 9722dd4b51e44ca99307736a149c5677(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:50,199 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:50,199 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/C, priority=12, startTime=1733202049595; duration=0sec 2024-12-03T05:00:50,199 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:50,199 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:C 2024-12-03T05:00:50,200 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dc7b395a55914facaa8ffcd292cbbdcf, entries=150, sequenceid=247, filesize=11.9 K 2024-12-03T05:00:50,202 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 9ba85df8e95baa2667d284a9d65f61b1 in 514ms, sequenceid=247, compaction requested=false 2024-12-03T05:00:50,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:50,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:50,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-03T05:00:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-03T05:00:50,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-03T05:00:50,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 674 msec 2024-12-03T05:00:50,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 682 msec 2024-12-03T05:00:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:50,237 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:50,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-03T05:00:50,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/9fbef852a4f242048f27c5daa9e0400d is 50, key is test_row_0/A:col10/1733202050231/Put/seqid=0 2024-12-03T05:00:50,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742125_1301 (size=12301) 2024-12-03T05:00:50,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/9fbef852a4f242048f27c5daa9e0400d 2024-12-03T05:00:50,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/feba6c8694de49b49d4949eef28f3029 is 50, key is test_row_0/B:col10/1733202050231/Put/seqid=0 2024-12-03T05:00:50,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202110318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202110322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742126_1302 (size=12301) 2024-12-03T05:00:50,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/feba6c8694de49b49d4949eef28f3029 2024-12-03T05:00:50,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/c0e098e1706b43618366beaae2fcc4a0 is 50, key is test_row_0/C:col10/1733202050231/Put/seqid=0 2024-12-03T05:00:50,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742127_1303 (size=12301) 2024-12-03T05:00:50,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/c0e098e1706b43618366beaae2fcc4a0 2024-12-03T05:00:50,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/9fbef852a4f242048f27c5daa9e0400d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/9fbef852a4f242048f27c5daa9e0400d 2024-12-03T05:00:50,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/9fbef852a4f242048f27c5daa9e0400d, entries=150, sequenceid=268, filesize=12.0 K 2024-12-03T05:00:50,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/feba6c8694de49b49d4949eef28f3029 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/feba6c8694de49b49d4949eef28f3029 2024-12-03T05:00:50,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/feba6c8694de49b49d4949eef28f3029, entries=150, sequenceid=268, filesize=12.0 K 2024-12-03T05:00:50,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/c0e098e1706b43618366beaae2fcc4a0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/c0e098e1706b43618366beaae2fcc4a0 2024-12-03T05:00:50,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/c0e098e1706b43618366beaae2fcc4a0, entries=150, sequenceid=268, filesize=12.0 K 2024-12-03T05:00:50,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 9ba85df8e95baa2667d284a9d65f61b1 in 154ms, sequenceid=268, compaction requested=true 2024-12-03T05:00:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:50,391 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:50,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:00:50,392 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:50,393 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:50,393 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/B is initiating minor compaction (all files) 2024-12-03T05:00:50,393 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/B in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:50,393 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a74ab0dcdfa04f979ce5f565d25e36eb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/61f6f283e59a4bcdbdd6846bf8d2e043, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/feba6c8694de49b49d4949eef28f3029] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=36.2 K 2024-12-03T05:00:50,393 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:50,393 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/A is initiating minor compaction (all files) 2024-12-03T05:00:50,393 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/A in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:50,394 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/16d10a74c0bf4ce7aeae0ae5da8c3817, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/f894cc958712457f8a1ea7d187855c06, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/9fbef852a4f242048f27c5daa9e0400d] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=36.2 K 2024-12-03T05:00:50,394 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a74ab0dcdfa04f979ce5f565d25e36eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733202047923 2024-12-03T05:00:50,394 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 16d10a74c0bf4ce7aeae0ae5da8c3817, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733202047923 2024-12-03T05:00:50,394 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting f894cc958712457f8a1ea7d187855c06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733202049120 2024-12-03T05:00:50,394 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61f6f283e59a4bcdbdd6846bf8d2e043, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733202049120 2024-12-03T05:00:50,395 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fbef852a4f242048f27c5daa9e0400d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733202049784 2024-12-03T05:00:50,395 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting feba6c8694de49b49d4949eef28f3029, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733202049784 2024-12-03T05:00:50,420 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:50,421 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/ecea4ac72e7f4df89312bc069c743a92 is 50, key is test_row_0/B:col10/1733202050231/Put/seqid=0 2024-12-03T05:00:50,423 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#A#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:50,424 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/5fbb97dbc74d40dea5af6d6541b98017 is 50, key is test_row_0/A:col10/1733202050231/Put/seqid=0 2024-12-03T05:00:50,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:50,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-03T05:00:50,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/ef0d8b30944d411bb6e7ce3e7239a5ad is 50, key is test_row_0/A:col10/1733202050301/Put/seqid=0 2024-12-03T05:00:50,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742129_1305 (size=12915) 2024-12-03T05:00:50,450 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/ecea4ac72e7f4df89312bc069c743a92 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/ecea4ac72e7f4df89312bc069c743a92 2024-12-03T05:00:50,455 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/B of 9ba85df8e95baa2667d284a9d65f61b1 into ecea4ac72e7f4df89312bc069c743a92(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:50,455 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:50,455 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/B, priority=13, startTime=1733202050391; duration=0sec 2024-12-03T05:00:50,455 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:50,455 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:B 2024-12-03T05:00:50,455 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:00:50,459 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:00:50,460 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/C is initiating minor compaction (all files) 2024-12-03T05:00:50,460 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/C in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:50,460 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/9722dd4b51e44ca99307736a149c5677, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dc7b395a55914facaa8ffcd292cbbdcf, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/c0e098e1706b43618366beaae2fcc4a0] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=36.2 K 2024-12-03T05:00:50,466 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9722dd4b51e44ca99307736a149c5677, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733202047923 2024-12-03T05:00:50,467 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc7b395a55914facaa8ffcd292cbbdcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733202049120 2024-12-03T05:00:50,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202110460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,468 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e098e1706b43618366beaae2fcc4a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733202049784 2024-12-03T05:00:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202110460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202110461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202110468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202110469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742130_1306 (size=14741) 2024-12-03T05:00:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742128_1304 (size=12915) 2024-12-03T05:00:50,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/ef0d8b30944d411bb6e7ce3e7239a5ad 2024-12-03T05:00:50,482 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#C#compaction#258 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:50,483 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/dcf32f4545a6468e88d4b274f16b7551 is 50, key is test_row_0/C:col10/1733202050231/Put/seqid=0 2024-12-03T05:00:50,489 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/5fbb97dbc74d40dea5af6d6541b98017 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/5fbb97dbc74d40dea5af6d6541b98017 2024-12-03T05:00:50,495 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/A of 9ba85df8e95baa2667d284a9d65f61b1 into 5fbb97dbc74d40dea5af6d6541b98017(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:50,495 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:50,495 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/A, priority=13, startTime=1733202050391; duration=0sec 2024-12-03T05:00:50,495 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:50,495 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:A 2024-12-03T05:00:50,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742131_1307 (size=12915) 2024-12-03T05:00:50,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/f4acda6be38e4518a7579ce3cf2c3c94 is 50, key is test_row_0/B:col10/1733202050301/Put/seqid=0 2024-12-03T05:00:50,520 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/dcf32f4545a6468e88d4b274f16b7551 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dcf32f4545a6468e88d4b274f16b7551 2024-12-03T05:00:50,531 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/C of 9ba85df8e95baa2667d284a9d65f61b1 into dcf32f4545a6468e88d4b274f16b7551(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:50,531 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:50,531 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/C, priority=13, startTime=1733202050391; duration=0sec 2024-12-03T05:00:50,532 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:50,532 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:C 2024-12-03T05:00:50,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742132_1308 (size=12301) 2024-12-03T05:00:50,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202110574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202110581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202110582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202110589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202110589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T05:00:50,671 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:50,671 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-03T05:00:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:00:50,674 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:50,675 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:50,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:00:50,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202110783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202110790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202110790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202110802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202110803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:50,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:00:50,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:50,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:50,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:50,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:50,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:50,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/f4acda6be38e4518a7579ce3cf2c3c94 2024-12-03T05:00:50,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/788ecd0f9d594ed5bd193d0b8f904983 is 50, key is test_row_0/C:col10/1733202050301/Put/seqid=0 2024-12-03T05:00:50,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742133_1309 (size=12301) 2024-12-03T05:00:50,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:00:50,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:50,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:50,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:50,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:50,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:00:51,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202111090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202111100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202111100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202111111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202111111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:00:51,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:51,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:51,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:51,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:51,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:51,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:00:51,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:51,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:51,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:51,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:51,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:00:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:51,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/788ecd0f9d594ed5bd193d0b8f904983 2024-12-03T05:00:51,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/ef0d8b30944d411bb6e7ce3e7239a5ad as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ef0d8b30944d411bb6e7ce3e7239a5ad 2024-12-03T05:00:51,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ef0d8b30944d411bb6e7ce3e7239a5ad, entries=200, sequenceid=288, filesize=14.4 K 2024-12-03T05:00:51,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/f4acda6be38e4518a7579ce3cf2c3c94 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/f4acda6be38e4518a7579ce3cf2c3c94 2024-12-03T05:00:51,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/f4acda6be38e4518a7579ce3cf2c3c94, entries=150, sequenceid=288, filesize=12.0 K 2024-12-03T05:00:51,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/788ecd0f9d594ed5bd193d0b8f904983 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/788ecd0f9d594ed5bd193d0b8f904983 2024-12-03T05:00:51,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/788ecd0f9d594ed5bd193d0b8f904983, entries=150, sequenceid=288, filesize=12.0 K 2024-12-03T05:00:51,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 9ba85df8e95baa2667d284a9d65f61b1 in 965ms, sequenceid=288, compaction requested=false 2024-12-03T05:00:51,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:51,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:00:51,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:51,454 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-03T05:00:51,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/2980d5dd87eb42139844fda28211b00a is 50, key is test_row_0/A:col10/1733202050459/Put/seqid=0 2024-12-03T05:00:51,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742134_1310 (size=12301) 2024-12-03T05:00:51,487 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/2980d5dd87eb42139844fda28211b00a 2024-12-03T05:00:51,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/5d28f7fea6d54a1bada13a85e2b78cee is 50, key is test_row_0/B:col10/1733202050459/Put/seqid=0 2024-12-03T05:00:51,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742135_1311 (size=12301) 2024-12-03T05:00:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:51,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:51,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202111638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202111639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202111640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202111640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202111649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202111751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202111751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202111751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202111751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202111761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:00:51,912 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/5d28f7fea6d54a1bada13a85e2b78cee 2024-12-03T05:00:51,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/dcbdbde58ac5467ebd44d77c4062ac9a is 50, key is test_row_0/C:col10/1733202050459/Put/seqid=0 2024-12-03T05:00:51,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742136_1312 (size=12301) 2024-12-03T05:00:51,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202111960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202111961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202111961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202111961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:51,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202111970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202112270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202112270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202112270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202112271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202112281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,324 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/dcbdbde58ac5467ebd44d77c4062ac9a 2024-12-03T05:00:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/2980d5dd87eb42139844fda28211b00a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2980d5dd87eb42139844fda28211b00a 2024-12-03T05:00:52,334 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2980d5dd87eb42139844fda28211b00a, entries=150, sequenceid=308, filesize=12.0 K 2024-12-03T05:00:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/5d28f7fea6d54a1bada13a85e2b78cee as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5d28f7fea6d54a1bada13a85e2b78cee 2024-12-03T05:00:52,357 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5d28f7fea6d54a1bada13a85e2b78cee, entries=150, sequenceid=308, filesize=12.0 K 2024-12-03T05:00:52,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/dcbdbde58ac5467ebd44d77c4062ac9a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dcbdbde58ac5467ebd44d77c4062ac9a 2024-12-03T05:00:52,364 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dcbdbde58ac5467ebd44d77c4062ac9a, entries=150, sequenceid=308, filesize=12.0 K 2024-12-03T05:00:52,365 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 9ba85df8e95baa2667d284a9d65f61b1 in 911ms, sequenceid=308, compaction requested=true 2024-12-03T05:00:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-03T05:00:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-03T05:00:52,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-03T05:00:52,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6920 sec 2024-12-03T05:00:52,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.6990 sec 2024-12-03T05:00:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:52,785 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:52,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-03T05:00:52,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/167f59a1e7b14f09ae543729885205c9 is 50, key is test_row_0/A:col10/1733202052783/Put/seqid=0 2024-12-03T05:00:52,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742137_1313 (size=14741) 2024-12-03T05:00:52,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/167f59a1e7b14f09ae543729885205c9 2024-12-03T05:00:52,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202112806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202112806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202112807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202112807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202112809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/d7784b724f9a40eaabcb7119311e32d8 is 50, key is test_row_0/B:col10/1733202052783/Put/seqid=0 2024-12-03T05:00:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:00:52,831 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:52,832 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:00:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-03T05:00:52,834 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:00:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:00:52,835 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:00:52,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:00:52,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742138_1314 (size=12301) 2024-12-03T05:00:52,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202112921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202112921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202112921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202112921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202112921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:00:52,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T05:00:52,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:52,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:52,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:52,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:52,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202113131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202113131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202113131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202113131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202113131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T05:00:53,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:53,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:00:53,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/d7784b724f9a40eaabcb7119311e32d8 2024-12-03T05:00:53,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/da85be903f80427ab24b0aae777ea916 is 50, key is test_row_0/C:col10/1733202052783/Put/seqid=0 2024-12-03T05:00:53,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742139_1315 (size=12301) 2024-12-03T05:00:53,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/da85be903f80427ab24b0aae777ea916 2024-12-03T05:00:53,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/167f59a1e7b14f09ae543729885205c9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/167f59a1e7b14f09ae543729885205c9 2024-12-03T05:00:53,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/167f59a1e7b14f09ae543729885205c9, entries=200, sequenceid=330, filesize=14.4 K 2024-12-03T05:00:53,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/d7784b724f9a40eaabcb7119311e32d8 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/d7784b724f9a40eaabcb7119311e32d8 2024-12-03T05:00:53,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/d7784b724f9a40eaabcb7119311e32d8, entries=150, sequenceid=330, filesize=12.0 K 2024-12-03T05:00:53,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/da85be903f80427ab24b0aae777ea916 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/da85be903f80427ab24b0aae777ea916 2024-12-03T05:00:53,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T05:00:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/da85be903f80427ab24b0aae777ea916, entries=150, sequenceid=330, filesize=12.0 K 2024-12-03T05:00:53,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 9ba85df8e95baa2667d284a9d65f61b1 in 522ms, sequenceid=330, compaction requested=true 2024-12-03T05:00:53,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:53,308 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:53,308 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:53,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:53,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:53,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:53,309 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:53,310 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/A is initiating minor compaction (all files) 2024-12-03T05:00:53,310 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/A in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,310 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/5fbb97dbc74d40dea5af6d6541b98017, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ef0d8b30944d411bb6e7ce3e7239a5ad, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2980d5dd87eb42139844fda28211b00a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/167f59a1e7b14f09ae543729885205c9] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=53.4 K 2024-12-03T05:00:53,310 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:53,310 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/B is initiating minor compaction (all files) 2024-12-03T05:00:53,310 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/B in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,311 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/ecea4ac72e7f4df89312bc069c743a92, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/f4acda6be38e4518a7579ce3cf2c3c94, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5d28f7fea6d54a1bada13a85e2b78cee, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/d7784b724f9a40eaabcb7119311e32d8] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=48.7 K 2024-12-03T05:00:53,311 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fbb97dbc74d40dea5af6d6541b98017, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733202049784 2024-12-03T05:00:53,311 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ecea4ac72e7f4df89312bc069c743a92, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733202049784 2024-12-03T05:00:53,311 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef0d8b30944d411bb6e7ce3e7239a5ad, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733202050299 2024-12-03T05:00:53,312 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting f4acda6be38e4518a7579ce3cf2c3c94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733202050301 2024-12-03T05:00:53,312 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2980d5dd87eb42139844fda28211b00a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733202050458 2024-12-03T05:00:53,312 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d28f7fea6d54a1bada13a85e2b78cee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733202050458 2024-12-03T05:00:53,313 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 167f59a1e7b14f09ae543729885205c9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733202051627 2024-12-03T05:00:53,313 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting d7784b724f9a40eaabcb7119311e32d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733202051627 2024-12-03T05:00:53,340 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:53,341 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/df875c50192b4a329a8ba7e024a306c8 is 50, key is test_row_0/B:col10/1733202052783/Put/seqid=0 2024-12-03T05:00:53,344 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#A#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:53,345 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/125608f2f10d4f6d94ef09bdcd40a440 is 50, key is test_row_0/A:col10/1733202052783/Put/seqid=0 2024-12-03T05:00:53,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742140_1316 (size=13051) 2024-12-03T05:00:53,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742141_1317 (size=13051) 2024-12-03T05:00:53,388 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/125608f2f10d4f6d94ef09bdcd40a440 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/125608f2f10d4f6d94ef09bdcd40a440 2024-12-03T05:00:53,399 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/A of 9ba85df8e95baa2667d284a9d65f61b1 into 125608f2f10d4f6d94ef09bdcd40a440(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:53,399 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:53,400 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/A, priority=12, startTime=1733202053308; duration=0sec 2024-12-03T05:00:53,400 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:53,400 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:A 2024-12-03T05:00:53,400 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:53,409 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:53,409 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/C is initiating minor compaction (all files) 2024-12-03T05:00:53,410 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/C in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,410 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dcf32f4545a6468e88d4b274f16b7551, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/788ecd0f9d594ed5bd193d0b8f904983, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dcbdbde58ac5467ebd44d77c4062ac9a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/da85be903f80427ab24b0aae777ea916] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=48.7 K 2024-12-03T05:00:53,410 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcf32f4545a6468e88d4b274f16b7551, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733202049784 2024-12-03T05:00:53,411 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 788ecd0f9d594ed5bd193d0b8f904983, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733202050301 2024-12-03T05:00:53,413 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcbdbde58ac5467ebd44d77c4062ac9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733202050458 2024-12-03T05:00:53,413 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting da85be903f80427ab24b0aae777ea916, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733202051627 2024-12-03T05:00:53,436 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:53,437 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/714e9ff1d65c479dbc32ac50f4e8e9b4 is 50, key is test_row_0/C:col10/1733202052783/Put/seqid=0 2024-12-03T05:00:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:53,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742142_1318 (size=13051) 2024-12-03T05:00:53,453 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:53,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-03T05:00:53,457 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/714e9ff1d65c479dbc32ac50f4e8e9b4 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/714e9ff1d65c479dbc32ac50f4e8e9b4 2024-12-03T05:00:53,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T05:00:53,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:53,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:00:53,474 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/C of 9ba85df8e95baa2667d284a9d65f61b1 into 714e9ff1d65c479dbc32ac50f4e8e9b4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:53,474 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:53,474 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/C, priority=12, startTime=1733202053309; duration=0sec 2024-12-03T05:00:53,474 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:53,474 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:C 2024-12-03T05:00:53,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/3ac307940faf47e0a91ecac382f70e0c is 50, key is test_row_0/A:col10/1733202053442/Put/seqid=0 2024-12-03T05:00:53,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742143_1319 (size=12301) 2024-12-03T05:00:53,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/3ac307940faf47e0a91ecac382f70e0c 2024-12-03T05:00:53,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202113494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202113495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202113497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202113498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202113498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/2105b0a7d9514299ba367558130470d8 is 50, key is test_row_0/B:col10/1733202053442/Put/seqid=0 2024-12-03T05:00:53,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742144_1320 (size=12301) 2024-12-03T05:00:53,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202113610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202113611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202113611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T05:00:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202113611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202113611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T05:00:53,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:53,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,772 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/df875c50192b4a329a8ba7e024a306c8 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/df875c50192b4a329a8ba7e024a306c8 2024-12-03T05:00:53,779 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/B of 9ba85df8e95baa2667d284a9d65f61b1 into df875c50192b4a329a8ba7e024a306c8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:53,779 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:53,779 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/B, priority=12, startTime=1733202053308; duration=0sec 2024-12-03T05:00:53,779 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:53,779 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:B 2024-12-03T05:00:53,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202113821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202113821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202113821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202113821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202113822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:53,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T05:00:53,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:53,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:53,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:00:53,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/2105b0a7d9514299ba367558130470d8 2024-12-03T05:00:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:00:53,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/ee662a598eaf4382b7e157ace9b11543 is 50, key is test_row_0/C:col10/1733202053442/Put/seqid=0 2024-12-03T05:00:53,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742145_1321 (size=12301) 2024-12-03T05:00:53,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/ee662a598eaf4382b7e157ace9b11543 2024-12-03T05:00:53,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/3ac307940faf47e0a91ecac382f70e0c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3ac307940faf47e0a91ecac382f70e0c 2024-12-03T05:00:53,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3ac307940faf47e0a91ecac382f70e0c, entries=150, sequenceid=350, filesize=12.0 K 2024-12-03T05:00:53,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/2105b0a7d9514299ba367558130470d8 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/2105b0a7d9514299ba367558130470d8 2024-12-03T05:00:54,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/2105b0a7d9514299ba367558130470d8, entries=150, sequenceid=350, filesize=12.0 K 2024-12-03T05:00:54,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/ee662a598eaf4382b7e157ace9b11543 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/ee662a598eaf4382b7e157ace9b11543 2024-12-03T05:00:54,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/ee662a598eaf4382b7e157ace9b11543, entries=150, sequenceid=350, filesize=12.0 K 2024-12-03T05:00:54,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 9ba85df8e95baa2667d284a9d65f61b1 in 571ms, sequenceid=350, compaction requested=false 2024-12-03T05:00:54,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:54,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T05:00:54,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:54,091 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-03T05:00:54,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/ece01de46cc348b489ec113901e6877a is 50, key is test_row_0/A:col10/1733202053495/Put/seqid=0 2024-12-03T05:00:54,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742146_1322 (size=12301) 2024-12-03T05:00:54,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. as already flushing 2024-12-03T05:00:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:54,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202114150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202114154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202114154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202114157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202114157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202114260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202114271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202114271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202114271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202114271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202114470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202114481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202114481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202114481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202114481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,517 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/ece01de46cc348b489ec113901e6877a 2024-12-03T05:00:54,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/a115dac170ab4068a3a3746aeed37ffa is 50, key is test_row_0/B:col10/1733202053495/Put/seqid=0 2024-12-03T05:00:54,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742147_1323 (size=12301) 2024-12-03T05:00:54,559 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/a115dac170ab4068a3a3746aeed37ffa 2024-12-03T05:00:54,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/b276e322cff44626a8781a95a3c8b9ae is 50, key is test_row_0/C:col10/1733202053495/Put/seqid=0 2024-12-03T05:00:54,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742148_1324 (size=12301) 2024-12-03T05:00:54,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202114781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202114790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202114790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202114790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202114791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:00:55,008 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/b276e322cff44626a8781a95a3c8b9ae 2024-12-03T05:00:55,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/ece01de46cc348b489ec113901e6877a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ece01de46cc348b489ec113901e6877a 2024-12-03T05:00:55,018 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ece01de46cc348b489ec113901e6877a, entries=150, sequenceid=370, filesize=12.0 K 2024-12-03T05:00:55,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/a115dac170ab4068a3a3746aeed37ffa as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a115dac170ab4068a3a3746aeed37ffa 2024-12-03T05:00:55,024 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a115dac170ab4068a3a3746aeed37ffa, entries=150, sequenceid=370, filesize=12.0 K 2024-12-03T05:00:55,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-03T05:00:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/b276e322cff44626a8781a95a3c8b9ae as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b276e322cff44626a8781a95a3c8b9ae 2024-12-03T05:00:55,030 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b276e322cff44626a8781a95a3c8b9ae, entries=150, sequenceid=370, filesize=12.0 K 2024-12-03T05:00:55,031 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 9ba85df8e95baa2667d284a9d65f61b1 in 940ms, sequenceid=370, compaction requested=true 2024-12-03T05:00:55,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:55,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:55,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-03T05:00:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-03T05:00:55,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-03T05:00:55,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1980 sec 2024-12-03T05:00:55,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.2040 sec 2024-12-03T05:00:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:55,303 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:55,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-03T05:00:55,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/6ca9c5dfcc0e414c8da3881b22175d1f is 50, key is test_row_0/A:col10/1733202054146/Put/seqid=0 2024-12-03T05:00:55,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742149_1325 (size=14741) 2024-12-03T05:00:55,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202115321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202115321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202115323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202115324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202115325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202115430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202115430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202115441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202115441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202115441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,473 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-12-03T05:00:55,473 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:55,473 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:55,473 INFO [Thread-1164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1164. 2024-12-03T05:00:55,473 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:55,473 DEBUG [Thread-1164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:55,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:55,473 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:55,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:55,473 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:55,474 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-12-03T05:00:55,474 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:55,474 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:55,474 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:55,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:55,477 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-12-03T05:00:55,477 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:55,477 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:55,477 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:55,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:55,477 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-12-03T05:00:55,477 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:55,477 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:55,478 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:55,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:55,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202115641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202115641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202115650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202115650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202115650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/6ca9c5dfcc0e414c8da3881b22175d1f 2024-12-03T05:00:55,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/db46b7a549b74a7b99034cb5adc02625 is 50, key is test_row_0/B:col10/1733202054146/Put/seqid=0 2024-12-03T05:00:55,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742150_1326 (size=12301) 2024-12-03T05:00:55,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202115951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202115951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202115960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202115960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202115960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:56,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/db46b7a549b74a7b99034cb5adc02625 2024-12-03T05:00:56,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/d8e75dcd51564c27ac1acbe582ee942e is 50, key is test_row_0/C:col10/1733202054146/Put/seqid=0 2024-12-03T05:00:56,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742151_1327 (size=12301) 2024-12-03T05:00:56,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:56,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55794 deadline: 1733202116460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55836 deadline: 1733202116460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55788 deadline: 1733202116471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55826 deadline: 1733202116471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:56,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:00:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55776 deadline: 1733202116471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 2024-12-03T05:00:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:00:56,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:00:56,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/d8e75dcd51564c27ac1acbe582ee942e 2024-12-03T05:00:56,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/6ca9c5dfcc0e414c8da3881b22175d1f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6ca9c5dfcc0e414c8da3881b22175d1f 2024-12-03T05:00:56,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6ca9c5dfcc0e414c8da3881b22175d1f, entries=200, sequenceid=390, filesize=14.4 K 2024-12-03T05:00:56,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/db46b7a549b74a7b99034cb5adc02625 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/db46b7a549b74a7b99034cb5adc02625 2024-12-03T05:00:56,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/db46b7a549b74a7b99034cb5adc02625, entries=150, sequenceid=390, filesize=12.0 K 2024-12-03T05:00:56,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/d8e75dcd51564c27ac1acbe582ee942e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d8e75dcd51564c27ac1acbe582ee942e 2024-12-03T05:00:56,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d8e75dcd51564c27ac1acbe582ee942e, entries=150, sequenceid=390, filesize=12.0 K 2024-12-03T05:00:56,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 9ba85df8e95baa2667d284a9d65f61b1 in 1261ms, sequenceid=390, compaction requested=true 2024-12-03T05:00:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:00:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:00:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:56,564 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9ba85df8e95baa2667d284a9d65f61b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:00:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:56,564 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:56,565 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:56,565 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:56,565 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/A is initiating minor compaction (all files) 2024-12-03T05:00:56,565 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/B is initiating minor compaction (all files) 2024-12-03T05:00:56,566 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/A in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:56,566 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/B in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:56,566 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/125608f2f10d4f6d94ef09bdcd40a440, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3ac307940faf47e0a91ecac382f70e0c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ece01de46cc348b489ec113901e6877a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6ca9c5dfcc0e414c8da3881b22175d1f] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=51.2 K 2024-12-03T05:00:56,566 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/df875c50192b4a329a8ba7e024a306c8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/2105b0a7d9514299ba367558130470d8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a115dac170ab4068a3a3746aeed37ffa, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/db46b7a549b74a7b99034cb5adc02625] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=48.8 K 2024-12-03T05:00:56,566 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 125608f2f10d4f6d94ef09bdcd40a440, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733202051627 2024-12-03T05:00:56,566 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting df875c50192b4a329a8ba7e024a306c8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733202051627 2024-12-03T05:00:56,566 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ac307940faf47e0a91ecac382f70e0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733202053442 2024-12-03T05:00:56,566 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2105b0a7d9514299ba367558130470d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733202053442 2024-12-03T05:00:56,567 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ece01de46cc348b489ec113901e6877a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733202053494 2024-12-03T05:00:56,567 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a115dac170ab4068a3a3746aeed37ffa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733202053494 2024-12-03T05:00:56,567 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ca9c5dfcc0e414c8da3881b22175d1f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733202054146 2024-12-03T05:00:56,567 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting db46b7a549b74a7b99034cb5adc02625, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733202054146 2024-12-03T05:00:56,576 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#A#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:56,577 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/62b84802fdb14f7d8082c8fcd866b588 is 50, key is test_row_0/A:col10/1733202054146/Put/seqid=0 2024-12-03T05:00:56,579 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#B#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:56,580 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/a88aa0824879473baabe4897ac7375bd is 50, key is test_row_0/B:col10/1733202054146/Put/seqid=0 2024-12-03T05:00:56,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742152_1328 (size=13187) 2024-12-03T05:00:56,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742153_1329 (size=13187) 2024-12-03T05:00:56,986 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/62b84802fdb14f7d8082c8fcd866b588 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/62b84802fdb14f7d8082c8fcd866b588 2024-12-03T05:00:56,991 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/a88aa0824879473baabe4897ac7375bd as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a88aa0824879473baabe4897ac7375bd 2024-12-03T05:00:56,992 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/A of 9ba85df8e95baa2667d284a9d65f61b1 into 62b84802fdb14f7d8082c8fcd866b588(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:56,992 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:56,992 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/A, priority=12, startTime=1733202056564; duration=0sec 2024-12-03T05:00:56,992 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:00:56,992 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:A 2024-12-03T05:00:56,993 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:00:56,995 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:00:56,995 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 9ba85df8e95baa2667d284a9d65f61b1/C is initiating minor compaction (all files) 2024-12-03T05:00:56,995 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ba85df8e95baa2667d284a9d65f61b1/C in TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:56,995 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/714e9ff1d65c479dbc32ac50f4e8e9b4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/ee662a598eaf4382b7e157ace9b11543, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b276e322cff44626a8781a95a3c8b9ae, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d8e75dcd51564c27ac1acbe582ee942e] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp, totalSize=48.8 K 2024-12-03T05:00:56,995 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 714e9ff1d65c479dbc32ac50f4e8e9b4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733202051627 2024-12-03T05:00:56,996 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee662a598eaf4382b7e157ace9b11543, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733202053442 2024-12-03T05:00:56,996 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b276e322cff44626a8781a95a3c8b9ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733202053494 2024-12-03T05:00:56,996 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8e75dcd51564c27ac1acbe582ee942e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733202054146 2024-12-03T05:00:56,998 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/B of 9ba85df8e95baa2667d284a9d65f61b1 into a88aa0824879473baabe4897ac7375bd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:56,998 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:56,998 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/B, priority=12, startTime=1733202056564; duration=0sec 2024-12-03T05:00:56,998 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:56,998 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:B 2024-12-03T05:00:57,006 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ba85df8e95baa2667d284a9d65f61b1#C#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:00:57,007 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/3bde7f3124534819b817ce220ef4da29 is 50, key is test_row_0/C:col10/1733202054146/Put/seqid=0 2024-12-03T05:00:57,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742154_1330 (size=13187) 2024-12-03T05:00:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:00:57,011 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:57,416 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/3bde7f3124534819b817ce220ef4da29 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/3bde7f3124534819b817ce220ef4da29 2024-12-03T05:00:57,421 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9ba85df8e95baa2667d284a9d65f61b1/C of 9ba85df8e95baa2667d284a9d65f61b1 into 3bde7f3124534819b817ce220ef4da29(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:00:57,421 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:57,421 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., storeName=9ba85df8e95baa2667d284a9d65f61b1/C, priority=12, startTime=1733202056564; duration=0sec 2024-12-03T05:00:57,421 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:00:57,421 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ba85df8e95baa2667d284a9d65f61b1:C 2024-12-03T05:00:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:57,472 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:00:57,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-03T05:00:57,472 INFO [Thread-1155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1155. 2024-12-03T05:00:57,472 DEBUG [Thread-1155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:57,472 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,472 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:57,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/a2338c72f5a94fc98b0ebb09d1e78b16 is 50, key is test_row_0/A:col10/1733202055321/Put/seqid=0 2024-12-03T05:00:57,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742155_1331 (size=12301) 2024-12-03T05:00:57,482 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-12-03T05:00:57,482 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:57,482 INFO [Thread-1153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1153. 2024-12-03T05:00:57,482 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,482 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-12-03T05:00:57,482 DEBUG [Thread-1153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:57,483 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:57,483 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,483 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,483 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,483 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,483 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:57,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:57,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:57,484 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-12-03T05:00:57,484 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:00:57,484 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,484 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2710 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2714 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2703 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2730 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2671 2024-12-03T05:00:57,485 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T05:00:57,485 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:00:57,485 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:00:57,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:00:57,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:00:57,487 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:00:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:00:57,490 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202057490"}]},"ts":"1733202057490"} 2024-12-03T05:00:57,493 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:00:57,493 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:00:57,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:00:57,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9ba85df8e95baa2667d284a9d65f61b1, UNASSIGN}] 2024-12-03T05:00:57,496 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9ba85df8e95baa2667d284a9d65f61b1, UNASSIGN 2024-12-03T05:00:57,497 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=9ba85df8e95baa2667d284a9d65f61b1, regionState=CLOSING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:00:57,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9ba85df8e95baa2667d284a9d65f61b1, UNASSIGN because future has completed 2024-12-03T05:00:57,498 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:00:57,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:00:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:00:57,650 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:57,650 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:00:57,651 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 9ba85df8e95baa2667d284a9d65f61b1, disabling compactions & flushes 2024-12-03T05:00:57,651 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:00:57,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/a2338c72f5a94fc98b0ebb09d1e78b16 2024-12-03T05:00:57,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/c06ec5d717da46b59bd17e9beabaea74 is 50, key is test_row_0/B:col10/1733202055321/Put/seqid=0 2024-12-03T05:00:57,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742156_1332 (size=12301) 2024-12-03T05:00:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:00:58,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/c06ec5d717da46b59bd17e9beabaea74 2024-12-03T05:00:58,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/a0062776962240b5a6b913c26b07c0e6 is 50, key is test_row_0/C:col10/1733202055321/Put/seqid=0 2024-12-03T05:00:58,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742157_1333 (size=12301) 2024-12-03T05:00:58,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/a0062776962240b5a6b913c26b07c0e6 2024-12-03T05:00:58,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/a2338c72f5a94fc98b0ebb09d1e78b16 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/a2338c72f5a94fc98b0ebb09d1e78b16 2024-12-03T05:00:58,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/a2338c72f5a94fc98b0ebb09d1e78b16, entries=150, sequenceid=410, filesize=12.0 K 2024-12-03T05:00:58,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/c06ec5d717da46b59bd17e9beabaea74 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/c06ec5d717da46b59bd17e9beabaea74 2024-12-03T05:00:58,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/c06ec5d717da46b59bd17e9beabaea74, entries=150, sequenceid=410, filesize=12.0 K 2024-12-03T05:00:58,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/a0062776962240b5a6b913c26b07c0e6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/a0062776962240b5a6b913c26b07c0e6 2024-12-03T05:00:58,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/a0062776962240b5a6b913c26b07c0e6, entries=150, sequenceid=410, filesize=12.0 K 2024-12-03T05:00:58,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=26.84 KB/27480 for 9ba85df8e95baa2667d284a9d65f61b1 in 859ms, sequenceid=410, compaction requested=false 2024-12-03T05:00:58,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9ba85df8e95baa2667d284a9d65f61b1: 2024-12-03T05:00:58,332 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:58,332 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:58,332 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. after waiting 0 ms 2024-12-03T05:00:58,332 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:58,332 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 9ba85df8e95baa2667d284a9d65f61b1 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-03T05:00:58,335 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/e7fa655e4a3f492a8309368ccf684ec6 is 50, key is test_row_0/A:col10/1733202057481/Put/seqid=0 2024-12-03T05:00:58,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742158_1334 (size=12301) 2024-12-03T05:00:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:00:58,743 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/e7fa655e4a3f492a8309368ccf684ec6 2024-12-03T05:00:58,749 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/22569e760ec641caa4a14b81f096bf7e is 50, key is test_row_0/B:col10/1733202057481/Put/seqid=0 2024-12-03T05:00:58,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742159_1335 (size=12301) 2024-12-03T05:00:59,153 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/22569e760ec641caa4a14b81f096bf7e 2024-12-03T05:00:59,159 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/7982f89631b64e15b765a844ccafccb9 is 50, key is test_row_0/C:col10/1733202057481/Put/seqid=0 2024-12-03T05:00:59,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742160_1336 (size=12301) 2024-12-03T05:00:59,166 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/7982f89631b64e15b765a844ccafccb9 2024-12-03T05:00:59,170 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/A/e7fa655e4a3f492a8309368ccf684ec6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/e7fa655e4a3f492a8309368ccf684ec6 2024-12-03T05:00:59,174 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/e7fa655e4a3f492a8309368ccf684ec6, entries=150, sequenceid=417, filesize=12.0 K 2024-12-03T05:00:59,174 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/B/22569e760ec641caa4a14b81f096bf7e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/22569e760ec641caa4a14b81f096bf7e 2024-12-03T05:00:59,177 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/22569e760ec641caa4a14b81f096bf7e, entries=150, sequenceid=417, filesize=12.0 K 2024-12-03T05:00:59,178 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/.tmp/C/7982f89631b64e15b765a844ccafccb9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/7982f89631b64e15b765a844ccafccb9 2024-12-03T05:00:59,181 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/7982f89631b64e15b765a844ccafccb9, entries=150, sequenceid=417, filesize=12.0 K 2024-12-03T05:00:59,182 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 9ba85df8e95baa2667d284a9d65f61b1 in 850ms, sequenceid=417, compaction requested=true 2024-12-03T05:00:59,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/78c70e371fa0484ba2ab107decace1b9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/003afb8521d9469ba87e4db4911f14c0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6977c3bcbd94450b8ddc604befb7dba3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/33bb65f8178e410ab19033d622adde57, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/e21ff7bac4e04af29a5327c5702d6e3c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2daf93f4a2df4a64af2cf218f8e3be2b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/32adccfe5d5e48008f80867b10b2f4f9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/5d789a235edd41c5b6268dc331a8bceb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3231ed4e11c14f70bb3f5958f938f24e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/37d367318b9f4070848edb3c3b7d6634, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/98a2e913de1e4461b67cfdeb8ba44af1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/57e4a954024c4ee99f69c526a0d20bf6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/65c51e3fb5e74c049deaf6676bf03da1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/719b90e54fa6465083dddf12c22a4500, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2d22d3059f50426ebb75a58eac5778d5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/16d10a74c0bf4ce7aeae0ae5da8c3817, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/f894cc958712457f8a1ea7d187855c06, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/5fbb97dbc74d40dea5af6d6541b98017, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/9fbef852a4f242048f27c5daa9e0400d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ef0d8b30944d411bb6e7ce3e7239a5ad, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2980d5dd87eb42139844fda28211b00a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/167f59a1e7b14f09ae543729885205c9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/125608f2f10d4f6d94ef09bdcd40a440, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3ac307940faf47e0a91ecac382f70e0c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ece01de46cc348b489ec113901e6877a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6ca9c5dfcc0e414c8da3881b22175d1f] to archive 2024-12-03T05:00:59,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:00:59,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/78c70e371fa0484ba2ab107decace1b9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/78c70e371fa0484ba2ab107decace1b9 2024-12-03T05:00:59,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/003afb8521d9469ba87e4db4911f14c0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/003afb8521d9469ba87e4db4911f14c0 2024-12-03T05:00:59,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6977c3bcbd94450b8ddc604befb7dba3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6977c3bcbd94450b8ddc604befb7dba3 2024-12-03T05:00:59,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/33bb65f8178e410ab19033d622adde57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/33bb65f8178e410ab19033d622adde57 2024-12-03T05:00:59,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/e21ff7bac4e04af29a5327c5702d6e3c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/e21ff7bac4e04af29a5327c5702d6e3c 2024-12-03T05:00:59,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2daf93f4a2df4a64af2cf218f8e3be2b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2daf93f4a2df4a64af2cf218f8e3be2b 2024-12-03T05:00:59,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/32adccfe5d5e48008f80867b10b2f4f9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/32adccfe5d5e48008f80867b10b2f4f9 2024-12-03T05:00:59,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/5d789a235edd41c5b6268dc331a8bceb to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/5d789a235edd41c5b6268dc331a8bceb 2024-12-03T05:00:59,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3231ed4e11c14f70bb3f5958f938f24e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3231ed4e11c14f70bb3f5958f938f24e 2024-12-03T05:00:59,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/37d367318b9f4070848edb3c3b7d6634 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/37d367318b9f4070848edb3c3b7d6634 2024-12-03T05:00:59,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/98a2e913de1e4461b67cfdeb8ba44af1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/98a2e913de1e4461b67cfdeb8ba44af1 2024-12-03T05:00:59,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/57e4a954024c4ee99f69c526a0d20bf6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/57e4a954024c4ee99f69c526a0d20bf6 2024-12-03T05:00:59,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/65c51e3fb5e74c049deaf6676bf03da1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/65c51e3fb5e74c049deaf6676bf03da1 2024-12-03T05:00:59,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/719b90e54fa6465083dddf12c22a4500 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/719b90e54fa6465083dddf12c22a4500 2024-12-03T05:00:59,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2d22d3059f50426ebb75a58eac5778d5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2d22d3059f50426ebb75a58eac5778d5 2024-12-03T05:00:59,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/16d10a74c0bf4ce7aeae0ae5da8c3817 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/16d10a74c0bf4ce7aeae0ae5da8c3817 2024-12-03T05:00:59,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/f894cc958712457f8a1ea7d187855c06 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/f894cc958712457f8a1ea7d187855c06 2024-12-03T05:00:59,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/5fbb97dbc74d40dea5af6d6541b98017 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/5fbb97dbc74d40dea5af6d6541b98017 2024-12-03T05:00:59,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/9fbef852a4f242048f27c5daa9e0400d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/9fbef852a4f242048f27c5daa9e0400d 2024-12-03T05:00:59,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ef0d8b30944d411bb6e7ce3e7239a5ad to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ef0d8b30944d411bb6e7ce3e7239a5ad 2024-12-03T05:00:59,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2980d5dd87eb42139844fda28211b00a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/2980d5dd87eb42139844fda28211b00a 2024-12-03T05:00:59,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/167f59a1e7b14f09ae543729885205c9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/167f59a1e7b14f09ae543729885205c9 2024-12-03T05:00:59,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/125608f2f10d4f6d94ef09bdcd40a440 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/125608f2f10d4f6d94ef09bdcd40a440 2024-12-03T05:00:59,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3ac307940faf47e0a91ecac382f70e0c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/3ac307940faf47e0a91ecac382f70e0c 2024-12-03T05:00:59,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ece01de46cc348b489ec113901e6877a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/ece01de46cc348b489ec113901e6877a 2024-12-03T05:00:59,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6ca9c5dfcc0e414c8da3881b22175d1f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/6ca9c5dfcc0e414c8da3881b22175d1f 2024-12-03T05:00:59,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/fd2bcbfd57884d2e9a10b2fa7aa6de59, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/6a5d5a1d1810411796a79856c7832c46, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a326c7b88034471e82a8f3b7251cada7, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/ed99b6ad8750434d898267a6c21ecbf6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5e1578c233534e3a9712bd541985cd39, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/0ef0033d9db940389ee65ae487885dd8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/26eaae7ff9b541d4a780cc3f7a441370, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5a27b19e132143fc8922cea4b2f3b15e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/1b973b3b05f049d6b48651a5467578d8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/dc2b26bcd9214e3ca0f3dff52c31f58f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/d60c403c3b22481aa403b249918460d0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/9d6e02a66751483e93cfdcfd2b1bffe3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/8e69274e76ba4237a35f523bcf19dda1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/7e318c9067cb439ca1913f3f37021a0f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a74ab0dcdfa04f979ce5f565d25e36eb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/3a2eda64270847698061e8e2442cde63, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/61f6f283e59a4bcdbdd6846bf8d2e043, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/ecea4ac72e7f4df89312bc069c743a92, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/feba6c8694de49b49d4949eef28f3029, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/f4acda6be38e4518a7579ce3cf2c3c94, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5d28f7fea6d54a1bada13a85e2b78cee, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/df875c50192b4a329a8ba7e024a306c8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/d7784b724f9a40eaabcb7119311e32d8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/2105b0a7d9514299ba367558130470d8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a115dac170ab4068a3a3746aeed37ffa, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/db46b7a549b74a7b99034cb5adc02625] to archive 2024-12-03T05:00:59,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:00:59,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/fd2bcbfd57884d2e9a10b2fa7aa6de59 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/fd2bcbfd57884d2e9a10b2fa7aa6de59 2024-12-03T05:00:59,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/6a5d5a1d1810411796a79856c7832c46 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/6a5d5a1d1810411796a79856c7832c46 2024-12-03T05:00:59,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a326c7b88034471e82a8f3b7251cada7 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a326c7b88034471e82a8f3b7251cada7 2024-12-03T05:00:59,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/ed99b6ad8750434d898267a6c21ecbf6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/ed99b6ad8750434d898267a6c21ecbf6 2024-12-03T05:00:59,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5e1578c233534e3a9712bd541985cd39 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5e1578c233534e3a9712bd541985cd39 2024-12-03T05:00:59,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/0ef0033d9db940389ee65ae487885dd8 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/0ef0033d9db940389ee65ae487885dd8 2024-12-03T05:00:59,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/26eaae7ff9b541d4a780cc3f7a441370 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/26eaae7ff9b541d4a780cc3f7a441370 2024-12-03T05:00:59,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5a27b19e132143fc8922cea4b2f3b15e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5a27b19e132143fc8922cea4b2f3b15e 2024-12-03T05:00:59,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/1b973b3b05f049d6b48651a5467578d8 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/1b973b3b05f049d6b48651a5467578d8 2024-12-03T05:00:59,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/dc2b26bcd9214e3ca0f3dff52c31f58f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/dc2b26bcd9214e3ca0f3dff52c31f58f 2024-12-03T05:00:59,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/d60c403c3b22481aa403b249918460d0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/d60c403c3b22481aa403b249918460d0 2024-12-03T05:00:59,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/9d6e02a66751483e93cfdcfd2b1bffe3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/9d6e02a66751483e93cfdcfd2b1bffe3 2024-12-03T05:00:59,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/8e69274e76ba4237a35f523bcf19dda1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/8e69274e76ba4237a35f523bcf19dda1 2024-12-03T05:00:59,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/7e318c9067cb439ca1913f3f37021a0f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/7e318c9067cb439ca1913f3f37021a0f 2024-12-03T05:00:59,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a74ab0dcdfa04f979ce5f565d25e36eb to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a74ab0dcdfa04f979ce5f565d25e36eb 2024-12-03T05:00:59,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/3a2eda64270847698061e8e2442cde63 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/3a2eda64270847698061e8e2442cde63 2024-12-03T05:00:59,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/61f6f283e59a4bcdbdd6846bf8d2e043 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/61f6f283e59a4bcdbdd6846bf8d2e043 2024-12-03T05:00:59,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/ecea4ac72e7f4df89312bc069c743a92 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/ecea4ac72e7f4df89312bc069c743a92 2024-12-03T05:00:59,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/feba6c8694de49b49d4949eef28f3029 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/feba6c8694de49b49d4949eef28f3029 2024-12-03T05:00:59,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/f4acda6be38e4518a7579ce3cf2c3c94 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/f4acda6be38e4518a7579ce3cf2c3c94 2024-12-03T05:00:59,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5d28f7fea6d54a1bada13a85e2b78cee to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/5d28f7fea6d54a1bada13a85e2b78cee 2024-12-03T05:00:59,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/df875c50192b4a329a8ba7e024a306c8 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/df875c50192b4a329a8ba7e024a306c8 2024-12-03T05:00:59,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/d7784b724f9a40eaabcb7119311e32d8 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/d7784b724f9a40eaabcb7119311e32d8 2024-12-03T05:00:59,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/2105b0a7d9514299ba367558130470d8 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/2105b0a7d9514299ba367558130470d8 2024-12-03T05:00:59,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a115dac170ab4068a3a3746aeed37ffa to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a115dac170ab4068a3a3746aeed37ffa 2024-12-03T05:00:59,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/db46b7a549b74a7b99034cb5adc02625 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/db46b7a549b74a7b99034cb5adc02625 2024-12-03T05:00:59,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/a4fce78bfe644c489bd15efeaea7f9e2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/22830f634e674abd93eb9b6ae9eb6f6b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d29b341fa06c4470be7143c9855a0caa, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/27c3bd63242d486fb15c5209f9ec7930, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/233f952e6c7f4429a93260f7dc6d8cff, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/17cedcf19cc349f49642afa543797b39, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/4e6a72349aa847f8943b71f6f6535d74, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8d90507f65804bc09f0fe7a4bba0bbed, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/13561547f38544c0938d94af9d57494e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b95c90223f7d4edbb3fe54a9b2c40122, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8b9d2e028084419299dad765cb38c8ae, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/25a63c7f643e4c2392d043232736d2c1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d110033cd6e0498b92806c6bbbdae8d4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/1dafaf11226f4e4caa7f0b54bd98dd55, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/9722dd4b51e44ca99307736a149c5677, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8c8a3ae858634a85b3594122017c6405, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dc7b395a55914facaa8ffcd292cbbdcf, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dcf32f4545a6468e88d4b274f16b7551, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/c0e098e1706b43618366beaae2fcc4a0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/788ecd0f9d594ed5bd193d0b8f904983, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dcbdbde58ac5467ebd44d77c4062ac9a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/714e9ff1d65c479dbc32ac50f4e8e9b4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/da85be903f80427ab24b0aae777ea916, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/ee662a598eaf4382b7e157ace9b11543, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b276e322cff44626a8781a95a3c8b9ae, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d8e75dcd51564c27ac1acbe582ee942e] to archive 2024-12-03T05:00:59,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:00:59,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/a4fce78bfe644c489bd15efeaea7f9e2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/a4fce78bfe644c489bd15efeaea7f9e2 2024-12-03T05:00:59,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/22830f634e674abd93eb9b6ae9eb6f6b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/22830f634e674abd93eb9b6ae9eb6f6b 2024-12-03T05:00:59,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d29b341fa06c4470be7143c9855a0caa to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d29b341fa06c4470be7143c9855a0caa 2024-12-03T05:00:59,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/27c3bd63242d486fb15c5209f9ec7930 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/27c3bd63242d486fb15c5209f9ec7930 2024-12-03T05:00:59,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/233f952e6c7f4429a93260f7dc6d8cff to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/233f952e6c7f4429a93260f7dc6d8cff 2024-12-03T05:00:59,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/17cedcf19cc349f49642afa543797b39 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/17cedcf19cc349f49642afa543797b39 2024-12-03T05:00:59,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/4e6a72349aa847f8943b71f6f6535d74 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/4e6a72349aa847f8943b71f6f6535d74 2024-12-03T05:00:59,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8d90507f65804bc09f0fe7a4bba0bbed to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8d90507f65804bc09f0fe7a4bba0bbed 2024-12-03T05:00:59,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/13561547f38544c0938d94af9d57494e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/13561547f38544c0938d94af9d57494e 2024-12-03T05:00:59,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b95c90223f7d4edbb3fe54a9b2c40122 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b95c90223f7d4edbb3fe54a9b2c40122 2024-12-03T05:00:59,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8b9d2e028084419299dad765cb38c8ae to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8b9d2e028084419299dad765cb38c8ae 2024-12-03T05:00:59,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/25a63c7f643e4c2392d043232736d2c1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/25a63c7f643e4c2392d043232736d2c1 2024-12-03T05:00:59,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d110033cd6e0498b92806c6bbbdae8d4 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d110033cd6e0498b92806c6bbbdae8d4 2024-12-03T05:00:59,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/1dafaf11226f4e4caa7f0b54bd98dd55 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/1dafaf11226f4e4caa7f0b54bd98dd55 2024-12-03T05:00:59,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/9722dd4b51e44ca99307736a149c5677 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/9722dd4b51e44ca99307736a149c5677 2024-12-03T05:00:59,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8c8a3ae858634a85b3594122017c6405 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/8c8a3ae858634a85b3594122017c6405 2024-12-03T05:00:59,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dc7b395a55914facaa8ffcd292cbbdcf to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dc7b395a55914facaa8ffcd292cbbdcf 2024-12-03T05:00:59,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dcf32f4545a6468e88d4b274f16b7551 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dcf32f4545a6468e88d4b274f16b7551 2024-12-03T05:00:59,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/c0e098e1706b43618366beaae2fcc4a0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/c0e098e1706b43618366beaae2fcc4a0 2024-12-03T05:00:59,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/788ecd0f9d594ed5bd193d0b8f904983 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/788ecd0f9d594ed5bd193d0b8f904983 2024-12-03T05:00:59,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dcbdbde58ac5467ebd44d77c4062ac9a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/dcbdbde58ac5467ebd44d77c4062ac9a 2024-12-03T05:00:59,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/714e9ff1d65c479dbc32ac50f4e8e9b4 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/714e9ff1d65c479dbc32ac50f4e8e9b4 2024-12-03T05:00:59,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/da85be903f80427ab24b0aae777ea916 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/da85be903f80427ab24b0aae777ea916 2024-12-03T05:00:59,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/ee662a598eaf4382b7e157ace9b11543 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/ee662a598eaf4382b7e157ace9b11543 2024-12-03T05:00:59,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b276e322cff44626a8781a95a3c8b9ae to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/b276e322cff44626a8781a95a3c8b9ae 2024-12-03T05:00:59,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d8e75dcd51564c27ac1acbe582ee942e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/d8e75dcd51564c27ac1acbe582ee942e 2024-12-03T05:00:59,276 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/recovered.edits/420.seqid, newMaxSeqId=420, maxSeqId=1 2024-12-03T05:00:59,277 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. 2024-12-03T05:00:59,277 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 9ba85df8e95baa2667d284a9d65f61b1: Waiting for close lock at 1733202057650Running coprocessor pre-close hooks at 1733202057650Disabling compacts and flushes for region at 1733202057650Disabling writes for close at 1733202058332 (+682 ms)Obtaining lock to block concurrent updates at 1733202058332Preparing flush snapshotting stores in 9ba85df8e95baa2667d284a9d65f61b1 at 1733202058332Finished memstore snapshotting TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1733202058332Flushing stores of TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1. at 1733202058333 (+1 ms)Flushing 9ba85df8e95baa2667d284a9d65f61b1/A: creating writer at 1733202058333Flushing 9ba85df8e95baa2667d284a9d65f61b1/A: appending metadata at 1733202058335 (+2 ms)Flushing 9ba85df8e95baa2667d284a9d65f61b1/A: closing flushed file at 1733202058335Flushing 9ba85df8e95baa2667d284a9d65f61b1/B: creating writer at 1733202058746 (+411 ms)Flushing 9ba85df8e95baa2667d284a9d65f61b1/B: appending metadata at 1733202058748 (+2 ms)Flushing 9ba85df8e95baa2667d284a9d65f61b1/B: closing flushed file at 1733202058748Flushing 9ba85df8e95baa2667d284a9d65f61b1/C: creating writer at 1733202059156 (+408 ms)Flushing 9ba85df8e95baa2667d284a9d65f61b1/C: appending metadata at 1733202059159 (+3 ms)Flushing 9ba85df8e95baa2667d284a9d65f61b1/C: closing flushed file at 1733202059159Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73cb2c71: reopening flushed file at 1733202059170 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24dbf4ce: reopening flushed file at 1733202059174 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28235d82: reopening flushed file at 1733202059178 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 9ba85df8e95baa2667d284a9d65f61b1 in 850ms, sequenceid=417, compaction requested=true at 1733202059182 (+4 ms)Writing region close event to WAL at 1733202059273 (+91 ms)Running coprocessor post-close hooks at 1733202059277 (+4 ms)Closed at 1733202059277 2024-12-03T05:00:59,279 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:59,280 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=9ba85df8e95baa2667d284a9d65f61b1, regionState=CLOSED 2024-12-03T05:00:59,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:00:59,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-03T05:00:59,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 9ba85df8e95baa2667d284a9d65f61b1, server=32beccd76b35,34719,1733201977102 in 1.7840 sec 2024-12-03T05:00:59,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-03T05:00:59,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9ba85df8e95baa2667d284a9d65f61b1, UNASSIGN in 1.7880 sec 2024-12-03T05:00:59,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-03T05:00:59,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7930 sec 2024-12-03T05:00:59,292 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202059292"}]},"ts":"1733202059292"} 2024-12-03T05:00:59,293 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T05:00:59,293 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T05:00:59,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8070 sec 2024-12-03T05:00:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:00:59,641 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:59,641 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T05:00:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:59,643 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T05:00:59,644 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:59,647 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:59,649 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/recovered.edits] 2024-12-03T05:00:59,652 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/62b84802fdb14f7d8082c8fcd866b588 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/62b84802fdb14f7d8082c8fcd866b588 2024-12-03T05:00:59,654 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/a2338c72f5a94fc98b0ebb09d1e78b16 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/a2338c72f5a94fc98b0ebb09d1e78b16 2024-12-03T05:00:59,655 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/e7fa655e4a3f492a8309368ccf684ec6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/A/e7fa655e4a3f492a8309368ccf684ec6 2024-12-03T05:00:59,657 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/22569e760ec641caa4a14b81f096bf7e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/22569e760ec641caa4a14b81f096bf7e 2024-12-03T05:00:59,658 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a88aa0824879473baabe4897ac7375bd to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/a88aa0824879473baabe4897ac7375bd 2024-12-03T05:00:59,659 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/c06ec5d717da46b59bd17e9beabaea74 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/B/c06ec5d717da46b59bd17e9beabaea74 2024-12-03T05:00:59,661 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/3bde7f3124534819b817ce220ef4da29 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/3bde7f3124534819b817ce220ef4da29 2024-12-03T05:00:59,662 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/7982f89631b64e15b765a844ccafccb9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/7982f89631b64e15b765a844ccafccb9 2024-12-03T05:00:59,663 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/a0062776962240b5a6b913c26b07c0e6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/C/a0062776962240b5a6b913c26b07c0e6 2024-12-03T05:00:59,665 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/recovered.edits/420.seqid to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1/recovered.edits/420.seqid 2024-12-03T05:00:59,666 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/9ba85df8e95baa2667d284a9d65f61b1 2024-12-03T05:00:59,666 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T05:00:59,669 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:59,672 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T05:00:59,674 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T05:00:59,675 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:59,675 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T05:00:59,676 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733202059675"}]},"ts":"9223372036854775807"} 2024-12-03T05:00:59,678 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T05:00:59,678 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9ba85df8e95baa2667d284a9d65f61b1, NAME => 'TestAcidGuarantees,,1733202033267.9ba85df8e95baa2667d284a9d65f61b1.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T05:00:59,678 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T05:00:59,678 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733202059678"}]},"ts":"9223372036854775807"} 2024-12-03T05:00:59,682 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T05:00:59,683 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:59,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-12-03T05:00:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T05:00:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T05:00:59,751 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:00:59,764 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 247), OpenFileDescriptor=453 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=857 (was 911), ProcessCount=11 (was 11), AvailableMemoryMB=3539 (was 4264) 2024-12-03T05:00:59,778 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=857, ProcessCount=11, AvailableMemoryMB=3538 2024-12-03T05:00:59,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:00:59,780 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:00:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T05:00:59,782 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:00:59,782 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:00:59,783 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-12-03T05:00:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T05:00:59,783 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:00:59,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742161_1337 (size=959) 2024-12-03T05:00:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T05:01:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T05:01:00,191 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d 2024-12-03T05:01:00,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742162_1338 (size=53) 2024-12-03T05:01:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T05:01:00,598 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:01:00,598 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing eb947ae323132c3b2ada531b3598951f, disabling compactions & flushes 2024-12-03T05:01:00,598 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:00,598 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:00,598 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. after waiting 0 ms 2024-12-03T05:01:00,598 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:00,598 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:00,598 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for eb947ae323132c3b2ada531b3598951f: Waiting for close lock at 1733202060598Disabling compacts and flushes for region at 1733202060598Disabling writes for close at 1733202060598Writing region close event to WAL at 1733202060598Closed at 1733202060598 2024-12-03T05:01:00,599 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:01:00,600 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733202060600"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733202060600"}]},"ts":"1733202060600"} 2024-12-03T05:01:00,602 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:01:00,602 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:01:00,603 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202060602"}]},"ts":"1733202060602"} 2024-12-03T05:01:00,604 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T05:01:00,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, ASSIGN}] 2024-12-03T05:01:00,605 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, ASSIGN 2024-12-03T05:01:00,606 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, ASSIGN; state=OFFLINE, location=32beccd76b35,34719,1733201977102; forceNewPlan=false, retain=false 2024-12-03T05:01:00,757 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=eb947ae323132c3b2ada531b3598951f, regionState=OPENING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:01:00,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, ASSIGN because future has completed 2024-12-03T05:01:00,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:01:00,920 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:00,920 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:01:00,920 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,920 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:01:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T05:01:00,921 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,921 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,922 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,924 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb947ae323132c3b2ada531b3598951f columnFamilyName A 2024-12-03T05:01:00,924 DEBUG [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:00,924 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(327): Store=eb947ae323132c3b2ada531b3598951f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:01:00,924 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,926 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb947ae323132c3b2ada531b3598951f columnFamilyName B 2024-12-03T05:01:00,926 DEBUG [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:00,926 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(327): Store=eb947ae323132c3b2ada531b3598951f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:01:00,926 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,928 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb947ae323132c3b2ada531b3598951f columnFamilyName C 2024-12-03T05:01:00,928 DEBUG [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:00,928 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(327): Store=eb947ae323132c3b2ada531b3598951f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:01:00,928 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,929 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,929 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,930 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,930 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,931 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:01:00,932 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,935 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:01:00,935 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened eb947ae323132c3b2ada531b3598951f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75093279, jitterRate=0.11897705495357513}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:01:00,935 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:00,936 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for eb947ae323132c3b2ada531b3598951f: Running coprocessor pre-open hook at 1733202060921Writing region info on filesystem at 1733202060921Initializing all the Stores at 1733202060921Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202060922 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202060922Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202060922Cleaning up temporary data from old regions at 1733202060930 (+8 ms)Running coprocessor post-open hooks at 1733202060935 (+5 ms)Region opened successfully at 1733202060936 (+1 ms) 2024-12-03T05:01:00,937 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., pid=87, masterSystemTime=1733202060916 2024-12-03T05:01:00,939 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:00,940 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:00,940 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=eb947ae323132c3b2ada531b3598951f, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:01:00,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:01:00,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-03T05:01:00,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 in 178 msec 2024-12-03T05:01:00,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-03T05:01:00,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, ASSIGN in 340 msec 2024-12-03T05:01:00,948 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:01:00,948 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202060948"}]},"ts":"1733202060948"} 2024-12-03T05:01:00,950 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T05:01:00,950 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:01:00,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-12-03T05:01:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T05:01:01,930 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:01,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:01,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:01,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:01,933 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4750c40a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:01,935 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ed8acf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:01,936 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:01,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:01,939 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:01:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,43423,1733201976038 2024-12-03T05:01:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b49d0e3 2024-12-03T05:01:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:01:01,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:01:01,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:01:01,942 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:01:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T05:01:01,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742163_1339 (size=995) 2024-12-03T05:01:02,357 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-03T05:01:02,357 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-03T05:01:02,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:01:02,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, REOPEN/MOVE}] 2024-12-03T05:01:02,363 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, REOPEN/MOVE 2024-12-03T05:01:02,364 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=eb947ae323132c3b2ada531b3598951f, regionState=CLOSING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:01:02,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, REOPEN/MOVE because future has completed 2024-12-03T05:01:02,366 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:01:02,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:01:02,519 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,519 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:01:02,519 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing eb947ae323132c3b2ada531b3598951f, disabling compactions & flushes 2024-12-03T05:01:02,519 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:02,519 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:02,519 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. after waiting 0 ms 2024-12-03T05:01:02,519 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:02,523 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T05:01:02,523 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:02,523 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for eb947ae323132c3b2ada531b3598951f: Waiting for close lock at 1733202062519Running coprocessor pre-close hooks at 1733202062519Disabling compacts and flushes for region at 1733202062519Disabling writes for close at 1733202062519Writing region close event to WAL at 1733202062519Running coprocessor post-close hooks at 1733202062523 (+4 ms)Closed at 1733202062523 2024-12-03T05:01:02,523 WARN [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: eb947ae323132c3b2ada531b3598951f to self. 2024-12-03T05:01:02,525 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,526 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=eb947ae323132c3b2ada531b3598951f, regionState=CLOSED 2024-12-03T05:01:02,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:01:02,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-03T05:01:02,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 in 163 msec 2024-12-03T05:01:02,536 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, REOPEN/MOVE; state=CLOSED, location=32beccd76b35,34719,1733201977102; forceNewPlan=false, retain=true 2024-12-03T05:01:02,687 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=eb947ae323132c3b2ada531b3598951f, regionState=OPENING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:01:02,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, REOPEN/MOVE because future has completed 2024-12-03T05:01:02,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:01:02,845 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:02,845 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:01:02,845 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,846 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:01:02,846 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,846 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,847 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,848 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb947ae323132c3b2ada531b3598951f columnFamilyName A 2024-12-03T05:01:02,849 DEBUG [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:02,849 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(327): Store=eb947ae323132c3b2ada531b3598951f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:01:02,850 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,850 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb947ae323132c3b2ada531b3598951f columnFamilyName B 2024-12-03T05:01:02,850 DEBUG [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:02,851 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(327): Store=eb947ae323132c3b2ada531b3598951f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:01:02,851 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,851 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb947ae323132c3b2ada531b3598951f columnFamilyName C 2024-12-03T05:01:02,851 DEBUG [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:02,851 INFO [StoreOpener-eb947ae323132c3b2ada531b3598951f-1 {}] regionserver.HStore(327): Store=eb947ae323132c3b2ada531b3598951f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:01:02,853 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,854 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,855 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,855 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,855 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,856 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:01:02,857 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,858 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened eb947ae323132c3b2ada531b3598951f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70476732, jitterRate=0.05018514394760132}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:01:02,858 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,858 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for eb947ae323132c3b2ada531b3598951f: Running coprocessor pre-open hook at 1733202062846Writing region info on filesystem at 1733202062846Initializing all the Stores at 1733202062846Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202062846Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202062847 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202062847Cleaning up temporary data from old regions at 1733202062855 (+8 ms)Running coprocessor post-open hooks at 1733202062858 (+3 ms)Region opened successfully at 1733202062858 2024-12-03T05:01:02,859 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., pid=92, masterSystemTime=1733202062842 2024-12-03T05:01:02,861 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:02,861 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:02,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=eb947ae323132c3b2ada531b3598951f, regionState=OPEN, openSeqNum=5, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:01:02,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:01:02,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-12-03T05:01:02,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 in 175 msec 2024-12-03T05:01:02,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-03T05:01:02,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, REOPEN/MOVE in 503 msec 2024-12-03T05:01:02,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-03T05:01:02,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-12-03T05:01:02,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 927 msec 2024-12-03T05:01:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-03T05:01:02,873 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:02,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:02,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:02,875 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bda395f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:02,876 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:02,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130ac2c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:02,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:02,878 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c2e8324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:02,879 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:02,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ffa0c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:02,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:02,881 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d36979e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:02,883 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:02,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5859c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:02,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:02,885 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:02,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:02,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:02,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47b0a270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:02,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:02,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:02,886 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dca3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:02,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:02,888 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fa95cb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:02,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0a7bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:02,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:02,891 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e84481, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:02,892 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:02,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1251d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:02,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:02,894 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:02,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:02,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784b99cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:02,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:02,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:02,895 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:02,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e60f877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:02,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:02,897 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e8c3da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:02,899 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aea0c08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:02,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:02,901 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c104ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:02,902 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:02,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c694af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:02,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:02,904 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7197ccea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:02,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:02,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:02,907 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb23bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:02,922 DEBUG [Thread-1541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:02,922 DEBUG [Thread-1545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:02,924 DEBUG [Thread-1541 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:02,924 DEBUG [Thread-1545 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:02,924 DEBUG [Thread-1541 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:02,924 DEBUG [Thread-1545 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:02,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:02,926 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:02,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:02,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:01:02,930 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:02,934 DEBUG [Thread-1554 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:02,934 DEBUG [Thread-1554 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:02,935 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:02,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:01:02,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:01:02,941 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:02,942 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:02,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:02,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:01:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:01:02,945 DEBUG [Thread-1558 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:02,945 DEBUG [Thread-1558 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:02,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:01:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-03T05:01:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T05:01:02,952 DEBUG [Thread-1552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:02,962 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:02,963 DEBUG [Thread-1556 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:02,963 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:02,964 DEBUG [Thread-1556 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:02,964 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:02,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:02,965 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:02,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:01:02,966 DEBUG [Thread-1543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:02,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:02,968 DEBUG [Thread-1556 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:01:02,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:01:02,974 DEBUG [Thread-1552 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:02,974 DEBUG [Thread-1552 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:02,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:01:02,983 DEBUG [Thread-1547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:02,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:01:03,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203488ffc05a6584181916934db51b890f6_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202062949/Put/seqid=0 2024-12-03T05:01:03,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742164_1340 (size=12154) 2024-12-03T05:01:03,050 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:03,054 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203488ffc05a6584181916934db51b890f6_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203488ffc05a6584181916934db51b890f6_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:03,055 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6f9310e2f5ec41b4a214c1aab8a36cd6, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:03,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6f9310e2f5ec41b4a214c1aab8a36cd6 is 175, key is test_row_0/A:col10/1733202062949/Put/seqid=0 2024-12-03T05:01:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T05:01:03,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742165_1341 (size=30955) 2024-12-03T05:01:03,067 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6f9310e2f5ec41b4a214c1aab8a36cd6 2024-12-03T05:01:03,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/8b736627260a47d5a005225c5f205bba is 50, key is test_row_0/B:col10/1733202062949/Put/seqid=0 2024-12-03T05:01:03,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T05:01:03,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:03,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:03,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:03,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742166_1342 (size=12001) 2024-12-03T05:01:03,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/8b736627260a47d5a005225c5f205bba 2024-12-03T05:01:03,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/59feb42b1f5f4cf89eb7ff933d5d90a8 is 50, key is test_row_0/C:col10/1733202062949/Put/seqid=0 2024-12-03T05:01:03,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202123145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202123146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202123150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742167_1343 (size=12001) 2024-12-03T05:01:03,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202123164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202123165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202123271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T05:01:03,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202123271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202123271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T05:01:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:03,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:03,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:03,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202123290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202123291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T05:01:03,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:03,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:03,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:03,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202123482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202123491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202123491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202123511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202123512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/59feb42b1f5f4cf89eb7ff933d5d90a8 2024-12-03T05:01:03,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6f9310e2f5ec41b4a214c1aab8a36cd6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6f9310e2f5ec41b4a214c1aab8a36cd6 2024-12-03T05:01:03,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6f9310e2f5ec41b4a214c1aab8a36cd6, entries=150, sequenceid=17, filesize=30.2 K 2024-12-03T05:01:03,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/8b736627260a47d5a005225c5f205bba as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/8b736627260a47d5a005225c5f205bba 2024-12-03T05:01:03,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/8b736627260a47d5a005225c5f205bba, entries=150, sequenceid=17, filesize=11.7 K 2024-12-03T05:01:03,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/59feb42b1f5f4cf89eb7ff933d5d90a8 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/59feb42b1f5f4cf89eb7ff933d5d90a8 2024-12-03T05:01:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T05:01:03,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T05:01:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:03,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:03,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/59feb42b1f5f4cf89eb7ff933d5d90a8, entries=150, sequenceid=17, filesize=11.7 K 2024-12-03T05:01:03,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for eb947ae323132c3b2ada531b3598951f in 632ms, sequenceid=17, compaction requested=false 2024-12-03T05:01:03,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:03,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T05:01:03,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:03,747 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-03T05:01:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039252cbfe2f6b447a9e2069fbd29962d8_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202063162/Put/seqid=0 2024-12-03T05:01:03,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742168_1344 (size=12154) 2024-12-03T05:01:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:03,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:03,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202123828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202123830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202123831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202123831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202123841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202123951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202123951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:03,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202123951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T05:01:04,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:04,179 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039252cbfe2f6b447a9e2069fbd29962d8_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039252cbfe2f6b447a9e2069fbd29962d8_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:04,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202124171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:04,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202124171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:04,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202124171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:04,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f3f48b6d925a4c2f9d0d23952a180df1, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:04,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f3f48b6d925a4c2f9d0d23952a180df1 is 175, key is test_row_0/A:col10/1733202063162/Put/seqid=0 2024-12-03T05:01:04,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742169_1345 (size=30955) 2024-12-03T05:01:04,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202124350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:04,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202124350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:04,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202124490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:04,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202124490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:04,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202124490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:04,594 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f3f48b6d925a4c2f9d0d23952a180df1 2024-12-03T05:01:04,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/499272f33d6c4186b2cf6ae4419fc873 is 50, key is test_row_0/B:col10/1733202063162/Put/seqid=0 2024-12-03T05:01:04,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742170_1346 (size=12001) 2024-12-03T05:01:04,623 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/499272f33d6c4186b2cf6ae4419fc873 2024-12-03T05:01:04,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/f9029180db13485b9315bf0ffb4f134f is 50, key is test_row_0/C:col10/1733202063162/Put/seqid=0 2024-12-03T05:01:04,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742171_1347 (size=12001) 2024-12-03T05:01:04,820 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:01:04,932 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:01:05,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202125011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:05,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202125011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:05,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202125011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:05,049 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/f9029180db13485b9315bf0ffb4f134f 2024-12-03T05:01:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f3f48b6d925a4c2f9d0d23952a180df1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f3f48b6d925a4c2f9d0d23952a180df1 2024-12-03T05:01:05,059 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f3f48b6d925a4c2f9d0d23952a180df1, entries=150, sequenceid=40, filesize=30.2 K 2024-12-03T05:01:05,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/499272f33d6c4186b2cf6ae4419fc873 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/499272f33d6c4186b2cf6ae4419fc873 2024-12-03T05:01:05,064 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/499272f33d6c4186b2cf6ae4419fc873, entries=150, sequenceid=40, filesize=11.7 K 2024-12-03T05:01:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/f9029180db13485b9315bf0ffb4f134f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/f9029180db13485b9315bf0ffb4f134f 2024-12-03T05:01:05,069 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/f9029180db13485b9315bf0ffb4f134f, entries=150, sequenceid=40, filesize=11.7 K 2024-12-03T05:01:05,070 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for eb947ae323132c3b2ada531b3598951f in 1324ms, sequenceid=40, compaction requested=false 2024-12-03T05:01:05,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:05,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:05,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-03T05:01:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-03T05:01:05,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-03T05:01:05,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1070 sec 2024-12-03T05:01:05,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.1340 sec 2024-12-03T05:01:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T05:01:05,111 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:05,111 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-03T05:01:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T05:01:05,113 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:05,114 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:05,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T05:01:05,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T05:01:05,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:05,268 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-03T05:01:05,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035753131302f848d7b05a201e5a31323d_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202063826/Put/seqid=0 2024-12-03T05:01:05,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742172_1348 (size=12154) 2024-12-03T05:01:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:05,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T05:01:05,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:05,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202125522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202125517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:05,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202125640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202125641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T05:01:05,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:05,762 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035753131302f848d7b05a201e5a31323d_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035753131302f848d7b05a201e5a31323d_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:05,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ba2c518d008b4b8b8880b08eda462702, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:05,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ba2c518d008b4b8b8880b08eda462702 is 175, key is test_row_0/A:col10/1733202063826/Put/seqid=0 2024-12-03T05:01:05,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742173_1349 (size=30955) 2024-12-03T05:01:05,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202125860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:05,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202125860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:06,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202126030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:06,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202126031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:06,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202126031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:06,039 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in eb947ae323132c3b2ada531b3598951f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:06,040 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in eb947ae323132c3b2ada531b3598951f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:06,041 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in eb947ae323132c3b2ada531b3598951f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:06,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:06,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202126170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:06,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202126170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:06,200 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ba2c518d008b4b8b8880b08eda462702 2024-12-03T05:01:06,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/d8518eae2215424f927e70eb72d3cd76 is 50, key is test_row_0/B:col10/1733202063826/Put/seqid=0 2024-12-03T05:01:06,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742174_1350 (size=12001) 2024-12-03T05:01:06,228 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/d8518eae2215424f927e70eb72d3cd76 2024-12-03T05:01:06,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/9f72847217884772b2b7d94cb9e050ba is 50, key is test_row_0/C:col10/1733202063826/Put/seqid=0 2024-12-03T05:01:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T05:01:06,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742175_1351 (size=12001) 2024-12-03T05:01:06,657 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/9f72847217884772b2b7d94cb9e050ba 2024-12-03T05:01:06,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ba2c518d008b4b8b8880b08eda462702 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ba2c518d008b4b8b8880b08eda462702 2024-12-03T05:01:06,669 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ba2c518d008b4b8b8880b08eda462702, entries=150, sequenceid=53, filesize=30.2 K 2024-12-03T05:01:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/d8518eae2215424f927e70eb72d3cd76 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/d8518eae2215424f927e70eb72d3cd76 2024-12-03T05:01:06,674 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/d8518eae2215424f927e70eb72d3cd76, entries=150, sequenceid=53, filesize=11.7 K 2024-12-03T05:01:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/9f72847217884772b2b7d94cb9e050ba as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/9f72847217884772b2b7d94cb9e050ba 2024-12-03T05:01:06,691 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/9f72847217884772b2b7d94cb9e050ba, entries=150, sequenceid=53, filesize=11.7 K 2024-12-03T05:01:06,693 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for eb947ae323132c3b2ada531b3598951f in 1425ms, sequenceid=53, compaction requested=true 2024-12-03T05:01:06,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:06,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:06,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-03T05:01:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-03T05:01:06,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-03T05:01:06,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5810 sec 2024-12-03T05:01:06,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.5870 sec 2024-12-03T05:01:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:06,705 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:06,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:01:06,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203654db027beb446a0924b6e80b4c4a743_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202066700/Put/seqid=0 2024-12-03T05:01:06,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742176_1352 (size=17034) 2024-12-03T05:01:06,803 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:06,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202126803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:06,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202126804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:06,829 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203654db027beb446a0924b6e80b4c4a743_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203654db027beb446a0924b6e80b4c4a743_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:06,830 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0504731b5ad644cfb2e13135b273518e, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:06,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0504731b5ad644cfb2e13135b273518e is 175, key is test_row_0/A:col10/1733202066700/Put/seqid=0 2024-12-03T05:01:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:06,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742177_1353 (size=48139) 2024-12-03T05:01:06,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202126932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:06,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202126965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:07,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202127150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:07,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:07,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:07,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202127181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T05:01:07,261 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:07,261 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-03T05:01:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T05:01:07,264 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:07,264 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:07,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:07,287 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0504731b5ad644cfb2e13135b273518e 2024-12-03T05:01:07,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/769524cf267847b8b454ba933e072eed is 50, key is test_row_0/B:col10/1733202066700/Put/seqid=0 2024-12-03T05:01:07,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742178_1354 (size=12001) 2024-12-03T05:01:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T05:01:07,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T05:01:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:07,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202127470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:07,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202127511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:07,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T05:01:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:07,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T05:01:07,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/769524cf267847b8b454ba933e072eed 2024-12-03T05:01:07,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/ffa9e0b51aed4427a188d608a27d8938 is 50, key is test_row_0/C:col10/1733202066700/Put/seqid=0 2024-12-03T05:01:07,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742179_1355 (size=12001) 2024-12-03T05:01:07,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T05:01:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:07,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T05:01:07,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T05:01:07,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:07,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:07,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:07,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:07,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202127980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:08,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:08,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202128030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:08,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T05:01:08,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:08,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:08,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:08,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:08,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:08,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:08,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:08,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202128051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:08,060 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in eb947ae323132c3b2ada531b3598951f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:08,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:08,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202128062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202128062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:08,071 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in eb947ae323132c3b2ada531b3598951f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:08,072 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in eb947ae323132c3b2ada531b3598951f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:08,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/ffa9e0b51aed4427a188d608a27d8938 2024-12-03T05:01:08,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0504731b5ad644cfb2e13135b273518e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0504731b5ad644cfb2e13135b273518e 2024-12-03T05:01:08,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0504731b5ad644cfb2e13135b273518e, entries=250, sequenceid=78, filesize=47.0 K 2024-12-03T05:01:08,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/769524cf267847b8b454ba933e072eed as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/769524cf267847b8b454ba933e072eed 2024-12-03T05:01:08,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/769524cf267847b8b454ba933e072eed, entries=150, sequenceid=78, filesize=11.7 K 2024-12-03T05:01:08,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/ffa9e0b51aed4427a188d608a27d8938 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ffa9e0b51aed4427a188d608a27d8938 2024-12-03T05:01:08,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ffa9e0b51aed4427a188d608a27d8938, entries=150, sequenceid=78, filesize=11.7 K 2024-12-03T05:01:08,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for eb947ae323132c3b2ada531b3598951f in 1442ms, sequenceid=78, compaction requested=true 2024-12-03T05:01:08,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:08,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:08,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:08,147 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:01:08,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:08,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:08,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:08,147 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:01:08,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:08,149 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:01:08,149 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/B is initiating minor compaction (all files) 2024-12-03T05:01:08,149 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:01:08,149 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/B in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:08,149 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/8b736627260a47d5a005225c5f205bba, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/499272f33d6c4186b2cf6ae4419fc873, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/d8518eae2215424f927e70eb72d3cd76, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/769524cf267847b8b454ba933e072eed] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=46.9 K 2024-12-03T05:01:08,149 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/A is initiating minor compaction (all files) 2024-12-03T05:01:08,150 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/A in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:08,150 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6f9310e2f5ec41b4a214c1aab8a36cd6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f3f48b6d925a4c2f9d0d23952a180df1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ba2c518d008b4b8b8880b08eda462702, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0504731b5ad644cfb2e13135b273518e] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=137.7 K 2024-12-03T05:01:08,150 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:08,150 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6f9310e2f5ec41b4a214c1aab8a36cd6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f3f48b6d925a4c2f9d0d23952a180df1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ba2c518d008b4b8b8880b08eda462702, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0504731b5ad644cfb2e13135b273518e] 2024-12-03T05:01:08,151 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f9310e2f5ec41b4a214c1aab8a36cd6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733202062949 2024-12-03T05:01:08,151 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b736627260a47d5a005225c5f205bba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733202062949 2024-12-03T05:01:08,152 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 499272f33d6c4186b2cf6ae4419fc873, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733202063144 2024-12-03T05:01:08,152 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3f48b6d925a4c2f9d0d23952a180df1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733202063144 2024-12-03T05:01:08,152 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting d8518eae2215424f927e70eb72d3cd76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733202063826 2024-12-03T05:01:08,152 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 769524cf267847b8b454ba933e072eed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733202065469 2024-12-03T05:01:08,153 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba2c518d008b4b8b8880b08eda462702, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733202063826 2024-12-03T05:01:08,153 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0504731b5ad644cfb2e13135b273518e, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733202065469 2024-12-03T05:01:08,168 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:08,168 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/5ba42c78c2934d2281b6d1c9016b3fbc is 50, key is test_row_0/B:col10/1733202066700/Put/seqid=0 2024-12-03T05:01:08,174 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:08,176 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203d09660278a2b466892486bbba3f7dd04_eb947ae323132c3b2ada531b3598951f store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:08,181 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203d09660278a2b466892486bbba3f7dd04_eb947ae323132c3b2ada531b3598951f, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:08,181 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d09660278a2b466892486bbba3f7dd04_eb947ae323132c3b2ada531b3598951f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:08,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742180_1356 (size=12139) 2024-12-03T05:01:08,191 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/5ba42c78c2934d2281b6d1c9016b3fbc as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/5ba42c78c2934d2281b6d1c9016b3fbc 2024-12-03T05:01:08,197 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb947ae323132c3b2ada531b3598951f/B of eb947ae323132c3b2ada531b3598951f into 5ba42c78c2934d2281b6d1c9016b3fbc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:08,197 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:08,197 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/B, priority=12, startTime=1733202068147; duration=0sec 2024-12-03T05:01:08,198 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:08,198 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:B 2024-12-03T05:01:08,198 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:01:08,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T05:01:08,203 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:01:08,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:08,203 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/C is initiating minor compaction (all files) 2024-12-03T05:01:08,203 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/C in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:08,203 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:01:08,204 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/59feb42b1f5f4cf89eb7ff933d5d90a8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/f9029180db13485b9315bf0ffb4f134f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/9f72847217884772b2b7d94cb9e050ba, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ffa9e0b51aed4427a188d608a27d8938] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=46.9 K 2024-12-03T05:01:08,204 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 59feb42b1f5f4cf89eb7ff933d5d90a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733202062949 2024-12-03T05:01:08,204 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting f9029180db13485b9315bf0ffb4f134f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733202063144 2024-12-03T05:01:08,206 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f72847217884772b2b7d94cb9e050ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733202063826 2024-12-03T05:01:08,207 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ffa9e0b51aed4427a188d608a27d8938, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733202065469 2024-12-03T05:01:08,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742181_1357 (size=4469) 2024-12-03T05:01:08,229 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:08,229 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/c8ab83cca4a24f7398dd2eb645932fef is 50, key is test_row_0/C:col10/1733202066700/Put/seqid=0 2024-12-03T05:01:08,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032c49c334305743ee9224d0c75fe0f7e4_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202066795/Put/seqid=0 2024-12-03T05:01:08,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742182_1358 (size=12139) 2024-12-03T05:01:08,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742183_1359 (size=12154) 2024-12-03T05:01:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T05:01:08,624 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#A#compaction#301 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:08,625 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/975e3834bcb146beb6ea6a33d5ac91ca is 175, key is test_row_0/A:col10/1733202066700/Put/seqid=0 2024-12-03T05:01:08,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742184_1360 (size=31093) 2024-12-03T05:01:08,635 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/975e3834bcb146beb6ea6a33d5ac91ca as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/975e3834bcb146beb6ea6a33d5ac91ca 2024-12-03T05:01:08,641 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb947ae323132c3b2ada531b3598951f/A of eb947ae323132c3b2ada531b3598951f into 975e3834bcb146beb6ea6a33d5ac91ca(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:08,641 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:08,641 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/A, priority=12, startTime=1733202068147; duration=0sec 2024-12-03T05:01:08,641 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:08,641 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:A 2024-12-03T05:01:08,669 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/c8ab83cca4a24f7398dd2eb645932fef as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/c8ab83cca4a24f7398dd2eb645932fef 2024-12-03T05:01:08,674 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb947ae323132c3b2ada531b3598951f/C of eb947ae323132c3b2ada531b3598951f into c8ab83cca4a24f7398dd2eb645932fef(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:08,674 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:08,674 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/C, priority=12, startTime=1733202068147; duration=0sec 2024-12-03T05:01:08,674 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:08,674 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:C 2024-12-03T05:01:08,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,688 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032c49c334305743ee9224d0c75fe0f7e4_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032c49c334305743ee9224d0c75fe0f7e4_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:08,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/750cec1724f8483cae35e51fc5bad4ff, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:08,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/750cec1724f8483cae35e51fc5bad4ff is 175, key is test_row_0/A:col10/1733202066795/Put/seqid=0 2024-12-03T05:01:08,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742185_1361 (size=30955) 2024-12-03T05:01:08,716 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/750cec1724f8483cae35e51fc5bad4ff 2024-12-03T05:01:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/872681070bf64aee92e5141aafef36b1 is 50, key is test_row_0/B:col10/1733202066795/Put/seqid=0 2024-12-03T05:01:08,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742186_1362 (size=12001) 2024-12-03T05:01:08,736 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/872681070bf64aee92e5141aafef36b1 2024-12-03T05:01:08,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/e409b64a3ae6453d817a0d43c881483a is 50, key is test_row_0/C:col10/1733202066795/Put/seqid=0 2024-12-03T05:01:08,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742187_1363 (size=12001) 2024-12-03T05:01:08,771 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/e409b64a3ae6453d817a0d43c881483a 2024-12-03T05:01:08,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/750cec1724f8483cae35e51fc5bad4ff as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/750cec1724f8483cae35e51fc5bad4ff 2024-12-03T05:01:08,785 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/750cec1724f8483cae35e51fc5bad4ff, entries=150, sequenceid=90, filesize=30.2 K 2024-12-03T05:01:08,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/872681070bf64aee92e5141aafef36b1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/872681070bf64aee92e5141aafef36b1 2024-12-03T05:01:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,790 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/872681070bf64aee92e5141aafef36b1, entries=150, sequenceid=90, filesize=11.7 K 2024-12-03T05:01:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/e409b64a3ae6453d817a0d43c881483a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/e409b64a3ae6453d817a0d43c881483a 2024-12-03T05:01:08,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,798 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/e409b64a3ae6453d817a0d43c881483a, entries=150, sequenceid=90, filesize=11.7 K 2024-12-03T05:01:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,800 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for eb947ae323132c3b2ada531b3598951f in 596ms, sequenceid=90, compaction requested=false 2024-12-03T05:01:08,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:08,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-03T05:01:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-03T05:01:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-03T05:01:08,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5380 sec 2024-12-03T05:01:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.5440 sec 2024-12-03T05:01:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,328 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:09,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:01:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:09,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203856a5270973f45f7b117829b9379f3ce_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202069325/Put/seqid=0 2024-12-03T05:01:09,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742188_1364 (size=14594) 2024-12-03T05:01:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T05:01:09,411 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:09,412 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-03T05:01:09,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:09,415 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:09,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:01:09,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202129467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:09,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202129474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:09,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:01:09,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:01:09,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:09,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:09,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:09,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:09,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:09,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202129590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202129590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:09,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:01:09,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:09,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:09,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:09,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:09,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:01:09,792 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:09,804 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203856a5270973f45f7b117829b9379f3ce_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203856a5270973f45f7b117829b9379f3ce_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:09,805 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0bc5cce5f26748b9935e13a566ba8107, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:09,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0bc5cce5f26748b9935e13a566ba8107 is 175, key is test_row_0/A:col10/1733202069325/Put/seqid=0 2024-12-03T05:01:09,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202129800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:09,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202129800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:09,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742189_1365 (size=39549) 2024-12-03T05:01:09,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:01:09,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:09,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:09,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:09,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:09,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:01:10,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:01:10,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:10,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:10,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202130121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:10,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202130121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:10,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:01:10,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,250 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0bc5cce5f26748b9935e13a566ba8107 2024-12-03T05:01:10,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/593e2067c1854c2e94f276ae48f41670 is 50, key is test_row_0/B:col10/1733202069325/Put/seqid=0 2024-12-03T05:01:10,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742190_1366 (size=12001) 2024-12-03T05:01:10,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:01:10,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:10,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:01:10,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:10,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:01:10,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202130640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:10,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202130641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:10,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:01:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/593e2067c1854c2e94f276ae48f41670 2024-12-03T05:01:10,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/1171d44197124b97b7d35723b87338da is 50, key is test_row_0/C:col10/1733202069325/Put/seqid=0 2024-12-03T05:01:10,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742191_1367 (size=12001) 2024-12-03T05:01:10,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:01:10,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:10,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:01:10,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:10,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:10,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:11,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/1171d44197124b97b7d35723b87338da 2024-12-03T05:01:11,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0bc5cce5f26748b9935e13a566ba8107 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0bc5cce5f26748b9935e13a566ba8107 2024-12-03T05:01:11,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0bc5cce5f26748b9935e13a566ba8107, entries=200, sequenceid=103, filesize=38.6 K 2024-12-03T05:01:11,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/593e2067c1854c2e94f276ae48f41670 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/593e2067c1854c2e94f276ae48f41670 2024-12-03T05:01:11,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/593e2067c1854c2e94f276ae48f41670, entries=150, sequenceid=103, filesize=11.7 K 2024-12-03T05:01:11,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/1171d44197124b97b7d35723b87338da as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/1171d44197124b97b7d35723b87338da 2024-12-03T05:01:11,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/1171d44197124b97b7d35723b87338da, entries=150, sequenceid=103, filesize=11.7 K 2024-12-03T05:01:11,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for eb947ae323132c3b2ada531b3598951f in 1791ms, sequenceid=103, compaction requested=true 2024-12-03T05:01:11,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:11,119 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:11,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:11,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:11,120 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:11,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:11,121 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:11,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:11,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:11,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:11,121 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/A is initiating minor compaction (all files) 2024-12-03T05:01:11,121 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/A in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:11,121 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/975e3834bcb146beb6ea6a33d5ac91ca, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/750cec1724f8483cae35e51fc5bad4ff, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0bc5cce5f26748b9935e13a566ba8107] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=99.2 K 2024-12-03T05:01:11,121 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:11,121 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/975e3834bcb146beb6ea6a33d5ac91ca, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/750cec1724f8483cae35e51fc5bad4ff, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0bc5cce5f26748b9935e13a566ba8107] 2024-12-03T05:01:11,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:01:11,122 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 975e3834bcb146beb6ea6a33d5ac91ca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733202065469 2024-12-03T05:01:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:11,122 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:01:11,122 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 750cec1724f8483cae35e51fc5bad4ff, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733202066772 2024-12-03T05:01:11,122 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:11,122 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/B is initiating minor compaction (all files) 2024-12-03T05:01:11,123 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/B in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:11,123 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc5cce5f26748b9935e13a566ba8107, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733202069200 2024-12-03T05:01:11,123 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/5ba42c78c2934d2281b6d1c9016b3fbc, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/872681070bf64aee92e5141aafef36b1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/593e2067c1854c2e94f276ae48f41670] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=35.3 K 2024-12-03T05:01:11,123 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ba42c78c2934d2281b6d1c9016b3fbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733202065469 2024-12-03T05:01:11,123 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 872681070bf64aee92e5141aafef36b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733202066772 2024-12-03T05:01:11,124 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 593e2067c1854c2e94f276ae48f41670, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733202069200 2024-12-03T05:01:11,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038e43bf971fc448349da9848d0bc1885e_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202069472/Put/seqid=0 2024-12-03T05:01:11,144 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#B#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:11,144 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/39dd52b3935d4f59a01bf2c9b6e80d54 is 50, key is test_row_0/B:col10/1733202069325/Put/seqid=0 2024-12-03T05:01:11,150 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:11,170 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120385f701e9999c46bca5a043792008984f_eb947ae323132c3b2ada531b3598951f store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:11,172 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120385f701e9999c46bca5a043792008984f_eb947ae323132c3b2ada531b3598951f, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:11,173 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120385f701e9999c46bca5a043792008984f_eb947ae323132c3b2ada531b3598951f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:11,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742192_1368 (size=12154) 2024-12-03T05:01:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:11,200 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038e43bf971fc448349da9848d0bc1885e_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038e43bf971fc448349da9848d0bc1885e_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ac605d7b23d44e2abe759adc43a0d79d, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:11,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ac605d7b23d44e2abe759adc43a0d79d is 175, key is test_row_0/A:col10/1733202069472/Put/seqid=0 2024-12-03T05:01:11,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742193_1369 (size=12241) 2024-12-03T05:01:11,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742194_1370 (size=4469) 2024-12-03T05:01:11,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742195_1371 (size=30955) 2024-12-03T05:01:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:01:11,612 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/39dd52b3935d4f59a01bf2c9b6e80d54 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/39dd52b3935d4f59a01bf2c9b6e80d54 2024-12-03T05:01:11,634 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/B of eb947ae323132c3b2ada531b3598951f into 39dd52b3935d4f59a01bf2c9b6e80d54(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:11,634 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:11,634 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/B, priority=13, startTime=1733202071120; duration=0sec 2024-12-03T05:01:11,634 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:11,634 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:B 2024-12-03T05:01:11,634 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:11,636 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:11,636 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/C is initiating minor compaction (all files) 2024-12-03T05:01:11,636 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/C in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:11,636 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/c8ab83cca4a24f7398dd2eb645932fef, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/e409b64a3ae6453d817a0d43c881483a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/1171d44197124b97b7d35723b87338da] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=35.3 K 2024-12-03T05:01:11,637 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c8ab83cca4a24f7398dd2eb645932fef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733202065469 2024-12-03T05:01:11,638 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting e409b64a3ae6453d817a0d43c881483a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733202066772 2024-12-03T05:01:11,638 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1171d44197124b97b7d35723b87338da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733202069200 2024-12-03T05:01:11,649 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#A#compaction#311 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:11,650 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/91588a1669f646948d625dd6c2aa2166 is 175, key is test_row_0/A:col10/1733202069325/Put/seqid=0 2024-12-03T05:01:11,658 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#C#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:11,658 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ac605d7b23d44e2abe759adc43a0d79d 2024-12-03T05:01:11,662 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/0abf964982a8480994dbad3c6a17036c is 50, key is test_row_0/C:col10/1733202069325/Put/seqid=0 2024-12-03T05:01:11,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742196_1372 (size=31195) 2024-12-03T05:01:11,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/0096bba05b764fb0ae8737edd4efc717 is 50, key is test_row_0/B:col10/1733202069472/Put/seqid=0 2024-12-03T05:01:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:11,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:11,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742197_1373 (size=12241) 2024-12-03T05:01:11,706 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/0abf964982a8480994dbad3c6a17036c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/0abf964982a8480994dbad3c6a17036c 2024-12-03T05:01:11,711 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/C of eb947ae323132c3b2ada531b3598951f into 0abf964982a8480994dbad3c6a17036c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:11,711 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:11,711 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/C, priority=13, startTime=1733202071121; duration=0sec 2024-12-03T05:01:11,711 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:11,711 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:C 2024-12-03T05:01:11,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742198_1374 (size=12001) 2024-12-03T05:01:11,714 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/0096bba05b764fb0ae8737edd4efc717 2024-12-03T05:01:11,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/97696ace5ffa4ea0ba5358fba8c52a4f is 50, key is test_row_0/C:col10/1733202069472/Put/seqid=0 2024-12-03T05:01:11,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202131738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:11,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742199_1375 (size=12001) 2024-12-03T05:01:11,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202131740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:11,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202131852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:11,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202131860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202132072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202132082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,091 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/91588a1669f646948d625dd6c2aa2166 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/91588a1669f646948d625dd6c2aa2166 2024-12-03T05:01:12,108 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/A of eb947ae323132c3b2ada531b3598951f into 91588a1669f646948d625dd6c2aa2166(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:12,108 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:12,108 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/A, priority=13, startTime=1733202071119; duration=0sec 2024-12-03T05:01:12,109 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:12,109 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:A 2024-12-03T05:01:12,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202132101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:12,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202132101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:12,112 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in eb947ae323132c3b2ada531b3598951f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8284 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,113 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in eb947ae323132c3b2ada531b3598951f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202132111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:12,126 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_0' in eb947ae323132c3b2ada531b3598951f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8284 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,142 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/97696ace5ffa4ea0ba5358fba8c52a4f 2024-12-03T05:01:12,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ac605d7b23d44e2abe759adc43a0d79d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ac605d7b23d44e2abe759adc43a0d79d 2024-12-03T05:01:12,155 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ac605d7b23d44e2abe759adc43a0d79d, entries=150, sequenceid=128, filesize=30.2 K 2024-12-03T05:01:12,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/0096bba05b764fb0ae8737edd4efc717 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/0096bba05b764fb0ae8737edd4efc717 2024-12-03T05:01:12,160 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/0096bba05b764fb0ae8737edd4efc717, entries=150, sequenceid=128, filesize=11.7 K 2024-12-03T05:01:12,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/97696ace5ffa4ea0ba5358fba8c52a4f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/97696ace5ffa4ea0ba5358fba8c52a4f 2024-12-03T05:01:12,165 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/97696ace5ffa4ea0ba5358fba8c52a4f, entries=150, sequenceid=128, filesize=11.7 K 2024-12-03T05:01:12,166 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for eb947ae323132c3b2ada531b3598951f in 1044ms, sequenceid=128, compaction requested=false 2024-12-03T05:01:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-03T05:01:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-03T05:01:12,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-03T05:01:12,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7520 sec 2024-12-03T05:01:12,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.7590 sec 2024-12-03T05:01:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:12,395 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:12,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-03T05:01:12,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203850590de449c4dd4825737b380730a60_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202071736/Put/seqid=0 2024-12-03T05:01:12,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742200_1376 (size=14794) 2024-12-03T05:01:12,421 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:12,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203850590de449c4dd4825737b380730a60_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203850590de449c4dd4825737b380730a60_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:12,429 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/2d40230fe56e497ab298b0139390e822, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:12,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/2d40230fe56e497ab298b0139390e822 is 175, key is test_row_0/A:col10/1733202071736/Put/seqid=0 2024-12-03T05:01:12,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742201_1377 (size=39749) 2024-12-03T05:01:12,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202132520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202132521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:12,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202132641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:12,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:12,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202132641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:12,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,833 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/2d40230fe56e497ab298b0139390e822 2024-12-03T05:01:12,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/30a318763ca2413c95c8b1179035b999 is 50, key is test_row_0/B:col10/1733202071736/Put/seqid=0 2024-12-03T05:01:12,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742202_1378 (size=12151) 2024-12-03T05:01:12,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/30a318763ca2413c95c8b1179035b999 2024-12-03T05:01:12,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/93a8d5c5d6914273854cc8ce7720e3d0 is 50, key is test_row_0/C:col10/1733202071736/Put/seqid=0 2024-12-03T05:01:12,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742203_1379 (size=12151) 2024-12-03T05:01:12,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/93a8d5c5d6914273854cc8ce7720e3d0 2024-12-03T05:01:12,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/2d40230fe56e497ab298b0139390e822 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/2d40230fe56e497ab298b0139390e822 2024-12-03T05:01:12,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/2d40230fe56e497ab298b0139390e822, entries=200, sequenceid=143, filesize=38.8 K 2024-12-03T05:01:12,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202132861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:12,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/30a318763ca2413c95c8b1179035b999 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/30a318763ca2413c95c8b1179035b999 2024-12-03T05:01:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202132861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:12,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/30a318763ca2413c95c8b1179035b999, entries=150, sequenceid=143, filesize=11.9 K 2024-12-03T05:01:12,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/93a8d5c5d6914273854cc8ce7720e3d0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/93a8d5c5d6914273854cc8ce7720e3d0 2024-12-03T05:01:12,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/93a8d5c5d6914273854cc8ce7720e3d0, entries=150, sequenceid=143, filesize=11.9 K 2024-12-03T05:01:12,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for eb947ae323132c3b2ada531b3598951f in 488ms, sequenceid=143, compaction requested=true 2024-12-03T05:01:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:12,883 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:12,883 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:12,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:12,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:12,884 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:12,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:12,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:12,884 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/A is initiating minor compaction (all files) 2024-12-03T05:01:12,884 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/A in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:12,884 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/91588a1669f646948d625dd6c2aa2166, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ac605d7b23d44e2abe759adc43a0d79d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/2d40230fe56e497ab298b0139390e822] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=99.5 K 2024-12-03T05:01:12,884 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:12,884 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/91588a1669f646948d625dd6c2aa2166, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ac605d7b23d44e2abe759adc43a0d79d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/2d40230fe56e497ab298b0139390e822] 2024-12-03T05:01:12,885 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91588a1669f646948d625dd6c2aa2166, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733202069200 2024-12-03T05:01:12,886 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:12,886 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/B is initiating minor compaction (all files) 2024-12-03T05:01:12,886 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/B in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:12,886 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/39dd52b3935d4f59a01bf2c9b6e80d54, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/0096bba05b764fb0ae8737edd4efc717, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/30a318763ca2413c95c8b1179035b999] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=35.5 K 2024-12-03T05:01:12,889 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 39dd52b3935d4f59a01bf2c9b6e80d54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733202069200 2024-12-03T05:01:12,889 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac605d7b23d44e2abe759adc43a0d79d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733202069463 2024-12-03T05:01:12,889 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0096bba05b764fb0ae8737edd4efc717, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733202069463 2024-12-03T05:01:12,890 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d40230fe56e497ab298b0139390e822, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733202071720 2024-12-03T05:01:12,890 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 30a318763ca2413c95c8b1179035b999, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733202071720 2024-12-03T05:01:12,898 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:12,898 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/c1ee555076ca432f97affc36195e5dc5 is 50, key is test_row_0/B:col10/1733202071736/Put/seqid=0 2024-12-03T05:01:12,901 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:12,908 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412036e6e89cd8a894a5b87a0ebb47c6474e2_eb947ae323132c3b2ada531b3598951f store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:12,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742204_1380 (size=12493) 2024-12-03T05:01:12,911 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412036e6e89cd8a894a5b87a0ebb47c6474e2_eb947ae323132c3b2ada531b3598951f, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:12,911 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036e6e89cd8a894a5b87a0ebb47c6474e2_eb947ae323132c3b2ada531b3598951f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:12,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742205_1381 (size=4469) 2024-12-03T05:01:12,924 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#A#compaction#319 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:12,925 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/029e25832c6a4432a78d33e83c305593 is 175, key is test_row_0/A:col10/1733202071736/Put/seqid=0 2024-12-03T05:01:12,926 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/c1ee555076ca432f97affc36195e5dc5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c1ee555076ca432f97affc36195e5dc5 2024-12-03T05:01:12,931 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/B of eb947ae323132c3b2ada531b3598951f into c1ee555076ca432f97affc36195e5dc5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:12,932 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:12,932 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/B, priority=13, startTime=1733202072883; duration=0sec 2024-12-03T05:01:12,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742206_1382 (size=31447) 2024-12-03T05:01:12,933 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:12,935 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:B 2024-12-03T05:01:12,935 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:12,938 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:12,938 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/C is initiating minor compaction (all files) 2024-12-03T05:01:12,938 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/C in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:12,938 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/0abf964982a8480994dbad3c6a17036c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/97696ace5ffa4ea0ba5358fba8c52a4f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/93a8d5c5d6914273854cc8ce7720e3d0] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=35.5 K 2024-12-03T05:01:12,938 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0abf964982a8480994dbad3c6a17036c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733202069200 2024-12-03T05:01:12,939 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 97696ace5ffa4ea0ba5358fba8c52a4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733202069463 2024-12-03T05:01:12,939 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 93a8d5c5d6914273854cc8ce7720e3d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733202071720 2024-12-03T05:01:12,951 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/029e25832c6a4432a78d33e83c305593 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/029e25832c6a4432a78d33e83c305593 2024-12-03T05:01:12,954 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:12,954 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/2f485684c74c4e1ab2caf5dc241ea46a is 50, key is test_row_0/C:col10/1733202071736/Put/seqid=0 2024-12-03T05:01:12,958 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/A of eb947ae323132c3b2ada531b3598951f into 029e25832c6a4432a78d33e83c305593(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:12,958 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:12,958 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/A, priority=13, startTime=1733202072883; duration=0sec 2024-12-03T05:01:12,958 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:12,958 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:A 2024-12-03T05:01:12,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742207_1383 (size=12493) 2024-12-03T05:01:13,187 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:13,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-03T05:01:13,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:13,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203152b02bac6944d4a88328f3c298a7059_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202072471/Put/seqid=0 2024-12-03T05:01:13,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202133213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:13,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742208_1384 (size=17284) 2024-12-03T05:01:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:13,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:13,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202133223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:13,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202133331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:13,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202133341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:13,389 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/2f485684c74c4e1ab2caf5dc241ea46a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/2f485684c74c4e1ab2caf5dc241ea46a 2024-12-03T05:01:13,397 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/C of eb947ae323132c3b2ada531b3598951f into 2f485684c74c4e1ab2caf5dc241ea46a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:13,397 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:13,397 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/C, priority=13, startTime=1733202072884; duration=0sec 2024-12-03T05:01:13,397 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:13,397 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:C 2024-12-03T05:01:13,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202133551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:13,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202133551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:01:13,574 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:13,574 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-03T05:01:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:01:13,578 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:13,579 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:13,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:13,626 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:13,635 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203152b02bac6944d4a88328f3c298a7059_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203152b02bac6944d4a88328f3c298a7059_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:13,636 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f8b2b2b0c964473b99d5c632d819e6af, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:13,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f8b2b2b0c964473b99d5c632d819e6af is 175, key is test_row_0/A:col10/1733202072471/Put/seqid=0 2024-12-03T05:01:13,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742209_1385 (size=48389) 2024-12-03T05:01:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:01:13,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:01:13,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:13,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:13,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:13,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:13,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202133861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:13,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202133861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:13,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:01:13,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:13,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:13,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:13,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:13,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:01:14,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:01:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:14,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:14,072 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f8b2b2b0c964473b99d5c632d819e6af 2024-12-03T05:01:14,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/022a765a484c4ed0ad0c860317f873b3 is 50, key is test_row_0/B:col10/1733202072471/Put/seqid=0 2024-12-03T05:01:14,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742210_1386 (size=12151) 2024-12-03T05:01:14,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/022a765a484c4ed0ad0c860317f873b3 2024-12-03T05:01:14,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/7a2e80f758644ad0ad6a3e6796c8d9b7 is 50, key is test_row_0/C:col10/1733202072471/Put/seqid=0 2024-12-03T05:01:14,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742211_1387 (size=12151) 2024-12-03T05:01:14,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/7a2e80f758644ad0ad6a3e6796c8d9b7 2024-12-03T05:01:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f8b2b2b0c964473b99d5c632d819e6af as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8b2b2b0c964473b99d5c632d819e6af 2024-12-03T05:01:14,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:01:14,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:14,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:14,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:14,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:01:14,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:14,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8b2b2b0c964473b99d5c632d819e6af, entries=250, sequenceid=171, filesize=47.3 K 2024-12-03T05:01:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/022a765a484c4ed0ad0c860317f873b3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/022a765a484c4ed0ad0c860317f873b3 2024-12-03T05:01:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/022a765a484c4ed0ad0c860317f873b3, entries=150, sequenceid=171, filesize=11.9 K 2024-12-03T05:01:14,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/7a2e80f758644ad0ad6a3e6796c8d9b7 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7a2e80f758644ad0ad6a3e6796c8d9b7 2024-12-03T05:01:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7a2e80f758644ad0ad6a3e6796c8d9b7, entries=150, sequenceid=171, filesize=11.9 K 2024-12-03T05:01:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for eb947ae323132c3b2ada531b3598951f in 1043ms, sequenceid=171, compaction requested=false 2024-12-03T05:01:14,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:01:14,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:14,371 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-03T05:01:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b1676d110a754ed0ae6382464e9524e2_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202073221/Put/seqid=0 2024-12-03T05:01:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742212_1388 (size=9814) 2024-12-03T05:01:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,450 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b1676d110a754ed0ae6382464e9524e2_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b1676d110a754ed0ae6382464e9524e2_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/564f132191ef4cfda8ad800e3da6a4a5, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/564f132191ef4cfda8ad800e3da6a4a5 is 175, key is test_row_0/A:col10/1733202073221/Put/seqid=0 2024-12-03T05:01:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742213_1389 (size=22461) 2024-12-03T05:01:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:14,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202134664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:14,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202134671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:01:14,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202134785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:14,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202134793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:14,885 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/564f132191ef4cfda8ad800e3da6a4a5 2024-12-03T05:01:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/85cd35b5341942c79348d6edeb9c8ee2 is 50, key is test_row_0/B:col10/1733202073221/Put/seqid=0 2024-12-03T05:01:14,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742214_1390 (size=9757) 2024-12-03T05:01:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202135001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:15,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202135010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:15,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202135310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:15,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202135320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:15,329 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/85cd35b5341942c79348d6edeb9c8ee2 2024-12-03T05:01:15,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/02f4d5c011c84e3ba9f9507d9b4d07bb is 50, key is test_row_0/C:col10/1733202073221/Put/seqid=0 2024-12-03T05:01:15,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742215_1391 (size=9757) 2024-12-03T05:01:15,359 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/02f4d5c011c84e3ba9f9507d9b4d07bb 2024-12-03T05:01:15,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/564f132191ef4cfda8ad800e3da6a4a5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/564f132191ef4cfda8ad800e3da6a4a5 2024-12-03T05:01:15,377 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/564f132191ef4cfda8ad800e3da6a4a5, entries=100, sequenceid=182, filesize=21.9 K 2024-12-03T05:01:15,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/85cd35b5341942c79348d6edeb9c8ee2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/85cd35b5341942c79348d6edeb9c8ee2 2024-12-03T05:01:15,386 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/85cd35b5341942c79348d6edeb9c8ee2, entries=100, sequenceid=182, filesize=9.5 K 2024-12-03T05:01:15,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/02f4d5c011c84e3ba9f9507d9b4d07bb as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/02f4d5c011c84e3ba9f9507d9b4d07bb 2024-12-03T05:01:15,396 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/02f4d5c011c84e3ba9f9507d9b4d07bb, entries=100, sequenceid=182, filesize=9.5 K 2024-12-03T05:01:15,397 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for eb947ae323132c3b2ada531b3598951f in 1027ms, sequenceid=182, compaction requested=true 2024-12-03T05:01:15,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:15,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:15,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-03T05:01:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-03T05:01:15,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-03T05:01:15,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8190 sec 2024-12-03T05:01:15,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.8370 sec 2024-12-03T05:01:15,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:01:15,740 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:15,741 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-03T05:01:15,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:01:15,749 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:15,750 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:15,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:15,829 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:15,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-03T05:01:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:01:15,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c216f880ffee47d98936ebcfcd03b2f0_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202074560/Put/seqid=0 2024-12-03T05:01:15,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742216_1392 (size=12304) 2024-12-03T05:01:15,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202135867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:15,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202135869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:15,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:01:15,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:15,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:15,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:15,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:15,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:15,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202135990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:15,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202135990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:16,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:01:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:01:16,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202136201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:16,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202136201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:16,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:01:16,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:16,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,281 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c216f880ffee47d98936ebcfcd03b2f0_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c216f880ffee47d98936ebcfcd03b2f0_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:16,281 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/70ea5d323c6f4334b7752a7a5979ba81, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:16,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/70ea5d323c6f4334b7752a7a5979ba81 is 175, key is test_row_0/A:col10/1733202074560/Put/seqid=0 2024-12-03T05:01:16,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742217_1393 (size=31105) 2024-12-03T05:01:16,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:01:16,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:16,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:01:16,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:01:16,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:16,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202136521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:16,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202136521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:16,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:16,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:16,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:16,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:01:16,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:16,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:16,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/70ea5d323c6f4334b7752a7a5979ba81 2024-12-03T05:01:16,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/a81d296d05e940f9967014e06d65dae1 is 50, key is test_row_0/B:col10/1733202074560/Put/seqid=0 2024-12-03T05:01:16,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742218_1394 (size=12151) 2024-12-03T05:01:16,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/a81d296d05e940f9967014e06d65dae1 2024-12-03T05:01:16,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/a1b631f0168e4b12830a07580a78f74c is 50, key is test_row_0/C:col10/1733202074560/Put/seqid=0 2024-12-03T05:01:16,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742219_1395 (size=12151) 2024-12-03T05:01:16,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/a1b631f0168e4b12830a07580a78f74c 2024-12-03T05:01:16,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/70ea5d323c6f4334b7752a7a5979ba81 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/70ea5d323c6f4334b7752a7a5979ba81 2024-12-03T05:01:16,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/70ea5d323c6f4334b7752a7a5979ba81, entries=150, sequenceid=209, filesize=30.4 K 2024-12-03T05:01:16,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/a81d296d05e940f9967014e06d65dae1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a81d296d05e940f9967014e06d65dae1 2024-12-03T05:01:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a81d296d05e940f9967014e06d65dae1, entries=150, sequenceid=209, filesize=11.9 K 2024-12-03T05:01:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/a1b631f0168e4b12830a07580a78f74c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/a1b631f0168e4b12830a07580a78f74c 2024-12-03T05:01:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/a1b631f0168e4b12830a07580a78f74c, entries=150, sequenceid=209, filesize=11.9 K 2024-12-03T05:01:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for eb947ae323132c3b2ada531b3598951f in 976ms, sequenceid=209, compaction requested=true 2024-12-03T05:01:16,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:16,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:16,806 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:01:16,806 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:01:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:16,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:16,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,807 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:01:16,807 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/B is initiating minor compaction (all files) 2024-12-03T05:01:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,807 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/B in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,807 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c1ee555076ca432f97affc36195e5dc5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/022a765a484c4ed0ad0c860317f873b3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/85cd35b5341942c79348d6edeb9c8ee2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a81d296d05e940f9967014e06d65dae1] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=45.5 K 2024-12-03T05:01:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,807 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133402 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:01:16,808 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/A is initiating minor compaction (all files) 2024-12-03T05:01:16,808 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c1ee555076ca432f97affc36195e5dc5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733202071720 2024-12-03T05:01:16,808 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/A in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,808 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/029e25832c6a4432a78d33e83c305593, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8b2b2b0c964473b99d5c632d819e6af, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/564f132191ef4cfda8ad800e3da6a4a5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/70ea5d323c6f4334b7752a7a5979ba81] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=130.3 K 2024-12-03T05:01:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,808 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,808 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/029e25832c6a4432a78d33e83c305593, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8b2b2b0c964473b99d5c632d819e6af, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/564f132191ef4cfda8ad800e3da6a4a5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/70ea5d323c6f4334b7752a7a5979ba81] 2024-12-03T05:01:16,808 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 022a765a484c4ed0ad0c860317f873b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733202072471 2024-12-03T05:01:16,808 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 029e25832c6a4432a78d33e83c305593, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733202071720 2024-12-03T05:01:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,808 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 85cd35b5341942c79348d6edeb9c8ee2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733202073205 2024-12-03T05:01:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,809 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8b2b2b0c964473b99d5c632d819e6af, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733202072471 2024-12-03T05:01:16,809 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a81d296d05e940f9967014e06d65dae1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733202074560 2024-12-03T05:01:16,809 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 564f132191ef4cfda8ad800e3da6a4a5, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733202073205 2024-12-03T05:01:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,810 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70ea5d323c6f4334b7752a7a5979ba81, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733202074560 2024-12-03T05:01:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,818 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,819 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,819 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/6a4f04f6e4534edf9c9681746f515da0 is 50, key is test_row_0/B:col10/1733202074560/Put/seqid=0 2024-12-03T05:01:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,821 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120349b8c767db6e45728c80df842f0e17a7_eb947ae323132c3b2ada531b3598951f store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,823 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120349b8c767db6e45728c80df842f0e17a7_eb947ae323132c3b2ada531b3598951f, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:16,823 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120349b8c767db6e45728c80df842f0e17a7_eb947ae323132c3b2ada531b3598951f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742220_1396 (size=12629) 2024-12-03T05:01:16,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742221_1397 (size=4469) 2024-12-03T05:01:16,830 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#A#compaction#331 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:16,830 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f63e04d6b8714a58bf6fe4a4fb4f7c5f is 175, key is test_row_0/A:col10/1733202074560/Put/seqid=0 2024-12-03T05:01:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:01:16,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,833 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-03T05:01:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742222_1398 (size=31583) 2024-12-03T05:01:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,842 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f63e04d6b8714a58bf6fe4a4fb4f7c5f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f63e04d6b8714a58bf6fe4a4fb4f7c5f 2024-12-03T05:01:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120389c636d2403c4e50b8191143048c0ff4_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202075854/Put/seqid=0 2024-12-03T05:01:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,862 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb947ae323132c3b2ada531b3598951f/A of eb947ae323132c3b2ada531b3598951f into f63e04d6b8714a58bf6fe4a4fb4f7c5f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:16,862 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:16,862 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/A, priority=12, startTime=1733202076805; duration=0sec 2024-12-03T05:01:16,862 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:16,863 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:A 2024-12-03T05:01:16,863 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:01:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,864 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:01:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,864 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/C is initiating minor compaction (all files) 2024-12-03T05:01:16,864 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/C in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,864 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/2f485684c74c4e1ab2caf5dc241ea46a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7a2e80f758644ad0ad6a3e6796c8d9b7, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/02f4d5c011c84e3ba9f9507d9b4d07bb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/a1b631f0168e4b12830a07580a78f74c] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=45.5 K 2024-12-03T05:01:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,864 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f485684c74c4e1ab2caf5dc241ea46a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733202071720 2024-12-03T05:01:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,865 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a2e80f758644ad0ad6a3e6796c8d9b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733202072471 2024-12-03T05:01:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,865 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02f4d5c011c84e3ba9f9507d9b4d07bb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733202073205 2024-12-03T05:01:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,866 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1b631f0168e4b12830a07580a78f74c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733202074560 2024-12-03T05:01:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:01:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742223_1399 (size=9814) 2024-12-03T05:01:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,908 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120389c636d2403c4e50b8191143048c0ff4_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389c636d2403c4e50b8191143048c0ff4_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,916 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#C#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,916 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/c4811770c4dc44b59bab1c6b8634a5be is 50, key is test_row_0/C:col10/1733202074560/Put/seqid=0 2024-12-03T05:01:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f8ea881d89d44f2ca8bf0163b8dc1acd, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f8ea881d89d44f2ca8bf0163b8dc1acd is 175, key is test_row_0/A:col10/1733202075854/Put/seqid=0 2024-12-03T05:01:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742224_1400 (size=12629) 2024-12-03T05:01:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742225_1401 (size=22461) 2024-12-03T05:01:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,937 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f8ea881d89d44f2ca8bf0163b8dc1acd 2024-12-03T05:01:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/1c4394bf57a04bbbaeba3a2e85e07fc3 is 50, key is test_row_0/B:col10/1733202075854/Put/seqid=0 2024-12-03T05:01:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742226_1402 (size=9757) 2024-12-03T05:01:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,948 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/1c4394bf57a04bbbaeba3a2e85e07fc3 2024-12-03T05:01:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/ff0899071c0a430d8fa029fc87228a19 is 50, key is test_row_0/C:col10/1733202075854/Put/seqid=0 2024-12-03T05:01:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742227_1403 (size=9757) 2024-12-03T05:01:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,963 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/ff0899071c0a430d8fa029fc87228a19 2024-12-03T05:01:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/f8ea881d89d44f2ca8bf0163b8dc1acd as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8ea881d89d44f2ca8bf0163b8dc1acd 2024-12-03T05:01:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,973 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8ea881d89d44f2ca8bf0163b8dc1acd, entries=100, sequenceid=218, filesize=21.9 K 2024-12-03T05:01:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/1c4394bf57a04bbbaeba3a2e85e07fc3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/1c4394bf57a04bbbaeba3a2e85e07fc3 2024-12-03T05:01:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,978 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/1c4394bf57a04bbbaeba3a2e85e07fc3, entries=100, sequenceid=218, filesize=9.5 K 2024-12-03T05:01:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/ff0899071c0a430d8fa029fc87228a19 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ff0899071c0a430d8fa029fc87228a19 2024-12-03T05:01:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,983 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ff0899071c0a430d8fa029fc87228a19, entries=100, sequenceid=218, filesize=9.5 K 2024-12-03T05:01:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,984 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for eb947ae323132c3b2ada531b3598951f in 151ms, sequenceid=218, compaction requested=false 2024-12-03T05:01:16,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:16,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:16,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-03T05:01:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-03T05:01:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-03T05:01:16,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2360 sec 2024-12-03T05:01:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.2480 sec 2024-12-03T05:01:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,239 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/6a4f04f6e4534edf9c9681746f515da0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/6a4f04f6e4534edf9c9681746f515da0 2024-12-03T05:01:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,244 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb947ae323132c3b2ada531b3598951f/B of eb947ae323132c3b2ada531b3598951f into 6a4f04f6e4534edf9c9681746f515da0(size=12.3 K), total size for store is 21.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:17,244 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:17,244 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/B, priority=12, startTime=1733202076806; duration=0sec 2024-12-03T05:01:17,244 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:17,244 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:B 2024-12-03T05:01:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,334 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/c4811770c4dc44b59bab1c6b8634a5be as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/c4811770c4dc44b59bab1c6b8634a5be 2024-12-03T05:01:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,339 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb947ae323132c3b2ada531b3598951f/C of eb947ae323132c3b2ada531b3598951f into c4811770c4dc44b59bab1c6b8634a5be(size=12.3 K), total size for store is 21.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:17,339 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:17,340 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/C, priority=12, startTime=1733202076806; duration=0sec 2024-12-03T05:01:17,340 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:17,340 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:C 2024-12-03T05:01:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:17,407 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:17,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:01:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203655fe06a1a2f4623be229d7491159949_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202077303/Put/seqid=0 2024-12-03T05:01:17,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742228_1404 (size=22268) 2024-12-03T05:01:17,486 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:17,490 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203655fe06a1a2f4623be229d7491159949_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203655fe06a1a2f4623be229d7491159949_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:17,494 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ca304a4b35914746835e9140e0fb37a8, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:17,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ca304a4b35914746835e9140e0fb37a8 is 175, key is test_row_0/A:col10/1733202077303/Put/seqid=0 2024-12-03T05:01:17,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742229_1405 (size=65673) 2024-12-03T05:01:17,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:17,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202137550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202137551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:17,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202137670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:17,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202137670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202137880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202137880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:01:17,901 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:17,901 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-03T05:01:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:01:17,903 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:17,904 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:17,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:17,910 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ca304a4b35914746835e9140e0fb37a8 2024-12-03T05:01:17,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/f177649326cb4697b1defd9bb0511b54 is 50, key is test_row_0/B:col10/1733202077303/Put/seqid=0 2024-12-03T05:01:17,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742230_1406 (size=12151) 2024-12-03T05:01:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:01:18,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:01:18,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:18,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202138191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:18,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202138200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:18,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:01:18,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:18,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:01:18,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/f177649326cb4697b1defd9bb0511b54 2024-12-03T05:01:18,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/aead914ca7504657a6feb96b4258af58 is 50, key is test_row_0/C:col10/1733202077303/Put/seqid=0 2024-12-03T05:01:18,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742231_1407 (size=12151) 2024-12-03T05:01:18,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:01:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:01:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:01:18,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:01:18,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:18,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:18,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202138710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:18,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202138720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:18,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/aead914ca7504657a6feb96b4258af58 2024-12-03T05:01:18,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ca304a4b35914746835e9140e0fb37a8 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ca304a4b35914746835e9140e0fb37a8 2024-12-03T05:01:18,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ca304a4b35914746835e9140e0fb37a8, entries=350, sequenceid=232, filesize=64.1 K 2024-12-03T05:01:18,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/f177649326cb4697b1defd9bb0511b54 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/f177649326cb4697b1defd9bb0511b54 2024-12-03T05:01:18,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/f177649326cb4697b1defd9bb0511b54, entries=150, sequenceid=232, filesize=11.9 K 2024-12-03T05:01:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/aead914ca7504657a6feb96b4258af58 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/aead914ca7504657a6feb96b4258af58 2024-12-03T05:01:18,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/aead914ca7504657a6feb96b4258af58, entries=150, sequenceid=232, filesize=11.9 K 2024-12-03T05:01:18,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for eb947ae323132c3b2ada531b3598951f in 1354ms, sequenceid=232, compaction requested=true 2024-12-03T05:01:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:18,761 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:01:18,763 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119717 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:18,763 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/A is initiating minor compaction (all files) 2024-12-03T05:01:18,763 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/A in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,763 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f63e04d6b8714a58bf6fe4a4fb4f7c5f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8ea881d89d44f2ca8bf0163b8dc1acd, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ca304a4b35914746835e9140e0fb37a8] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=116.9 K 2024-12-03T05:01:18,763 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,763 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f63e04d6b8714a58bf6fe4a4fb4f7c5f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8ea881d89d44f2ca8bf0163b8dc1acd, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ca304a4b35914746835e9140e0fb37a8] 2024-12-03T05:01:18,764 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f63e04d6b8714a58bf6fe4a4fb4f7c5f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733202074560 2024-12-03T05:01:18,764 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8ea881d89d44f2ca8bf0163b8dc1acd, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733202075854 2024-12-03T05:01:18,764 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca304a4b35914746835e9140e0fb37a8, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733202077169 2024-12-03T05:01:18,765 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:18,767 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:18,767 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/B is initiating minor compaction (all files) 2024-12-03T05:01:18,767 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/B in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,767 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/6a4f04f6e4534edf9c9681746f515da0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/1c4394bf57a04bbbaeba3a2e85e07fc3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/f177649326cb4697b1defd9bb0511b54] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=33.7 K 2024-12-03T05:01:18,768 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a4f04f6e4534edf9c9681746f515da0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733202074560 2024-12-03T05:01:18,768 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c4394bf57a04bbbaeba3a2e85e07fc3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733202075854 2024-12-03T05:01:18,769 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting f177649326cb4697b1defd9bb0511b54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733202077303 2024-12-03T05:01:18,779 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:18,798 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#B#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:18,799 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/dfc68c54d30349eea7219d16218cf990 is 50, key is test_row_0/B:col10/1733202077303/Put/seqid=0 2024-12-03T05:01:18,800 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203b113071f4b034c0bb7dbefb69f30d292_eb947ae323132c3b2ada531b3598951f store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:18,802 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203b113071f4b034c0bb7dbefb69f30d292_eb947ae323132c3b2ada531b3598951f, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:18,802 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b113071f4b034c0bb7dbefb69f30d292_eb947ae323132c3b2ada531b3598951f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:18,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:01:18,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742233_1409 (size=4469) 2024-12-03T05:01:18,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:18,830 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:01:18,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742232_1408 (size=12731) 2024-12-03T05:01:18,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120351b142fab61e4634bfdd578cd85a2d3a_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202077547/Put/seqid=0 2024-12-03T05:01:18,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742234_1410 (size=12304) 2024-12-03T05:01:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:01:19,241 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#A#compaction#339 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:19,242 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/e217d26e4ea84e248a78d362d70e2341 is 175, key is test_row_0/A:col10/1733202077303/Put/seqid=0 2024-12-03T05:01:19,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742235_1411 (size=31685) 2024-12-03T05:01:19,271 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/dfc68c54d30349eea7219d16218cf990 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/dfc68c54d30349eea7219d16218cf990 2024-12-03T05:01:19,283 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/B of eb947ae323132c3b2ada531b3598951f into dfc68c54d30349eea7219d16218cf990(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:19,283 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:19,283 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/B, priority=13, startTime=1733202078761; duration=0sec 2024-12-03T05:01:19,283 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:19,283 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:B 2024-12-03T05:01:19,283 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:19,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:19,285 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:19,285 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/C is initiating minor compaction (all files) 2024-12-03T05:01:19,286 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/e217d26e4ea84e248a78d362d70e2341 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/e217d26e4ea84e248a78d362d70e2341 2024-12-03T05:01:19,286 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/C in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:19,286 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/c4811770c4dc44b59bab1c6b8634a5be, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ff0899071c0a430d8fa029fc87228a19, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/aead914ca7504657a6feb96b4258af58] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=33.7 K 2024-12-03T05:01:19,289 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120351b142fab61e4634bfdd578cd85a2d3a_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120351b142fab61e4634bfdd578cd85a2d3a_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:19,290 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c4811770c4dc44b59bab1c6b8634a5be, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733202074560 2024-12-03T05:01:19,290 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ff0899071c0a430d8fa029fc87228a19, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733202075854 2024-12-03T05:01:19,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0ffca7d28c6a4fea95793d797ecfa4f5, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:19,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0ffca7d28c6a4fea95793d797ecfa4f5 is 175, key is test_row_0/A:col10/1733202077547/Put/seqid=0 2024-12-03T05:01:19,292 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting aead914ca7504657a6feb96b4258af58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733202077303 2024-12-03T05:01:19,295 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/A of eb947ae323132c3b2ada531b3598951f into e217d26e4ea84e248a78d362d70e2341(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:19,295 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:19,295 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/A, priority=13, startTime=1733202078761; duration=0sec 2024-12-03T05:01:19,295 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:19,295 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:A 2024-12-03T05:01:19,315 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:19,316 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/3adfafa55e17423eb663dca97b89e406 is 50, key is test_row_0/C:col10/1733202077303/Put/seqid=0 2024-12-03T05:01:19,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742236_1412 (size=31105) 2024-12-03T05:01:19,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742237_1413 (size=12731) 2024-12-03T05:01:19,392 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/3adfafa55e17423eb663dca97b89e406 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/3adfafa55e17423eb663dca97b89e406 2024-12-03T05:01:19,397 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/C of eb947ae323132c3b2ada531b3598951f into 3adfafa55e17423eb663dca97b89e406(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:19,397 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:19,397 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/C, priority=13, startTime=1733202078761; duration=0sec 2024-12-03T05:01:19,397 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:19,397 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:C 2024-12-03T05:01:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:19,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:19,760 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0ffca7d28c6a4fea95793d797ecfa4f5 2024-12-03T05:01:19,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/a372f7e05abf43688857a9f242a5aef2 is 50, key is test_row_0/B:col10/1733202077547/Put/seqid=0 2024-12-03T05:01:19,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742238_1414 (size=12151) 2024-12-03T05:01:19,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202139761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:19,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202139766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:19,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202139880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:19,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202139881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:01:20,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202140091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:20,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202140091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:20,189 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/a372f7e05abf43688857a9f242a5aef2 2024-12-03T05:01:20,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/280428ab73534df6951498ab75465cdb is 50, key is test_row_0/C:col10/1733202077547/Put/seqid=0 2024-12-03T05:01:20,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742239_1415 (size=12151) 2024-12-03T05:01:20,286 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/280428ab73534df6951498ab75465cdb 2024-12-03T05:01:20,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/0ffca7d28c6a4fea95793d797ecfa4f5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0ffca7d28c6a4fea95793d797ecfa4f5 2024-12-03T05:01:20,319 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0ffca7d28c6a4fea95793d797ecfa4f5, entries=150, sequenceid=257, filesize=30.4 K 2024-12-03T05:01:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/a372f7e05abf43688857a9f242a5aef2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a372f7e05abf43688857a9f242a5aef2 2024-12-03T05:01:20,338 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a372f7e05abf43688857a9f242a5aef2, entries=150, sequenceid=257, filesize=11.9 K 2024-12-03T05:01:20,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/280428ab73534df6951498ab75465cdb as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/280428ab73534df6951498ab75465cdb 2024-12-03T05:01:20,362 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/280428ab73534df6951498ab75465cdb, entries=150, sequenceid=257, filesize=11.9 K 2024-12-03T05:01:20,365 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for eb947ae323132c3b2ada531b3598951f in 1535ms, sequenceid=257, compaction requested=false 2024-12-03T05:01:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-03T05:01:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-03T05:01:20,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-03T05:01:20,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4660 sec 2024-12-03T05:01:20,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.4820 sec 2024-12-03T05:01:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:20,412 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:20,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-03T05:01:20,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f1d83c23e80547d8978a04df216f72d9_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202079764/Put/seqid=0 2024-12-03T05:01:20,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742240_1416 (size=14994) 2024-12-03T05:01:20,459 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:20,463 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f1d83c23e80547d8978a04df216f72d9_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f1d83c23e80547d8978a04df216f72d9_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:20,464 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6d95cdb3135944ecbd20a5ff49174da9, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:20,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6d95cdb3135944ecbd20a5ff49174da9 is 175, key is test_row_0/A:col10/1733202079764/Put/seqid=0 2024-12-03T05:01:20,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742241_1417 (size=39949) 2024-12-03T05:01:20,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202140549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:20,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202140551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:20,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202140670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:20,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202140671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:20,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:20,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:20,886 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6d95cdb3135944ecbd20a5ff49174da9 2024-12-03T05:01:20,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202140880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:20,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:20,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202140891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:20,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/80355a2b86754981800f60a9de729aad is 50, key is test_row_0/B:col10/1733202079764/Put/seqid=0 2024-12-03T05:01:20,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742242_1418 (size=12301) 2024-12-03T05:01:21,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202141201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:21,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202141210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:21,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/80355a2b86754981800f60a9de729aad 2024-12-03T05:01:21,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/609776b0b76140b4b921ea87cec40a22 is 50, key is test_row_0/C:col10/1733202079764/Put/seqid=0 2024-12-03T05:01:21,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742243_1419 (size=12301) 2024-12-03T05:01:21,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/609776b0b76140b4b921ea87cec40a22 2024-12-03T05:01:21,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6d95cdb3135944ecbd20a5ff49174da9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6d95cdb3135944ecbd20a5ff49174da9 2024-12-03T05:01:21,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6d95cdb3135944ecbd20a5ff49174da9, entries=200, sequenceid=272, filesize=39.0 K 2024-12-03T05:01:21,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/80355a2b86754981800f60a9de729aad as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/80355a2b86754981800f60a9de729aad 2024-12-03T05:01:21,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/80355a2b86754981800f60a9de729aad, entries=150, sequenceid=272, filesize=12.0 K 2024-12-03T05:01:21,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/609776b0b76140b4b921ea87cec40a22 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/609776b0b76140b4b921ea87cec40a22 2024-12-03T05:01:21,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/609776b0b76140b4b921ea87cec40a22, entries=150, sequenceid=272, filesize=12.0 K 2024-12-03T05:01:21,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for eb947ae323132c3b2ada531b3598951f in 1063ms, sequenceid=272, compaction requested=true 2024-12-03T05:01:21,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:21,475 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:21,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:21,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:21,475 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:21,486 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:21,486 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/A is initiating minor compaction (all files) 2024-12-03T05:01:21,486 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/A in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:21,486 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/e217d26e4ea84e248a78d362d70e2341, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0ffca7d28c6a4fea95793d797ecfa4f5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6d95cdb3135944ecbd20a5ff49174da9] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=100.3 K 2024-12-03T05:01:21,486 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:21,486 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/e217d26e4ea84e248a78d362d70e2341, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0ffca7d28c6a4fea95793d797ecfa4f5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6d95cdb3135944ecbd20a5ff49174da9] 2024-12-03T05:01:21,486 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:21,486 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/B is initiating minor compaction (all files) 2024-12-03T05:01:21,486 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/B in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:21,486 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/dfc68c54d30349eea7219d16218cf990, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a372f7e05abf43688857a9f242a5aef2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/80355a2b86754981800f60a9de729aad] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=36.3 K 2024-12-03T05:01:21,490 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting dfc68c54d30349eea7219d16218cf990, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733202077303 2024-12-03T05:01:21,490 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e217d26e4ea84e248a78d362d70e2341, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733202077303 2024-12-03T05:01:21,493 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a372f7e05abf43688857a9f242a5aef2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733202077531 2024-12-03T05:01:21,494 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ffca7d28c6a4fea95793d797ecfa4f5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733202077531 2024-12-03T05:01:21,494 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 80355a2b86754981800f60a9de729aad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733202079744 2024-12-03T05:01:21,495 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d95cdb3135944ecbd20a5ff49174da9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733202079736 2024-12-03T05:01:21,507 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:21,508 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/ee0d2d1ff05449caa9f63500e472800d is 50, key is test_row_0/B:col10/1733202079764/Put/seqid=0 2024-12-03T05:01:21,510 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:21,521 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203bc94293863364022b9f6ba02d192567c_eb947ae323132c3b2ada531b3598951f store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:21,523 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203bc94293863364022b9f6ba02d192567c_eb947ae323132c3b2ada531b3598951f, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:21,523 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bc94293863364022b9f6ba02d192567c_eb947ae323132c3b2ada531b3598951f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:21,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742244_1420 (size=12983) 2024-12-03T05:01:21,575 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/ee0d2d1ff05449caa9f63500e472800d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/ee0d2d1ff05449caa9f63500e472800d 2024-12-03T05:01:21,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742245_1421 (size=4469) 2024-12-03T05:01:21,605 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/B of eb947ae323132c3b2ada531b3598951f into ee0d2d1ff05449caa9f63500e472800d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:21,606 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:21,606 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/B, priority=13, startTime=1733202081475; duration=0sec 2024-12-03T05:01:21,606 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:21,606 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:B 2024-12-03T05:01:21,606 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:21,608 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:21,608 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/C is initiating minor compaction (all files) 2024-12-03T05:01:21,608 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/C in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:21,608 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/3adfafa55e17423eb663dca97b89e406, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/280428ab73534df6951498ab75465cdb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/609776b0b76140b4b921ea87cec40a22] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=36.3 K 2024-12-03T05:01:21,608 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3adfafa55e17423eb663dca97b89e406, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733202077303 2024-12-03T05:01:21,609 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 280428ab73534df6951498ab75465cdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733202077531 2024-12-03T05:01:21,610 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 609776b0b76140b4b921ea87cec40a22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733202079744 2024-12-03T05:01:21,621 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:21,622 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/698463f3862e45bbae4d32ab5ee023a1 is 50, key is test_row_0/C:col10/1733202079764/Put/seqid=0 2024-12-03T05:01:21,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742246_1422 (size=12983) 2024-12-03T05:01:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:21,729 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:21,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:01:21,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d1844324b15f46d4a73bc37c7a5ae2cc_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202081724/Put/seqid=0 2024-12-03T05:01:21,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742247_1423 (size=14994) 2024-12-03T05:01:21,748 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:21,762 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d1844324b15f46d4a73bc37c7a5ae2cc_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d1844324b15f46d4a73bc37c7a5ae2cc_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:21,765 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/cbacd21f184441bc9182a4a77293c4f1, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:21,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/cbacd21f184441bc9182a4a77293c4f1 is 175, key is test_row_0/A:col10/1733202081724/Put/seqid=0 2024-12-03T05:01:21,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202141771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:21,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742248_1424 (size=39949) 2024-12-03T05:01:21,791 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/cbacd21f184441bc9182a4a77293c4f1 2024-12-03T05:01:21,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202141785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:21,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/50d422317e964cd28ad7ecd43c275ed9 is 50, key is test_row_0/B:col10/1733202081724/Put/seqid=0 2024-12-03T05:01:21,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742249_1425 (size=12301) 2024-12-03T05:01:21,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/50d422317e964cd28ad7ecd43c275ed9 2024-12-03T05:01:21,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202141890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:21,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/602a06f9ccbf40908ddbddaebf21731f is 50, key is test_row_0/C:col10/1733202081724/Put/seqid=0 2024-12-03T05:01:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:21,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202141905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:21,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742250_1426 (size=12301) 2024-12-03T05:01:21,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/602a06f9ccbf40908ddbddaebf21731f 2024-12-03T05:01:21,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/cbacd21f184441bc9182a4a77293c4f1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/cbacd21f184441bc9182a4a77293c4f1 2024-12-03T05:01:21,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/cbacd21f184441bc9182a4a77293c4f1, entries=200, sequenceid=299, filesize=39.0 K 2024-12-03T05:01:21,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/50d422317e964cd28ad7ecd43c275ed9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/50d422317e964cd28ad7ecd43c275ed9 2024-12-03T05:01:21,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/50d422317e964cd28ad7ecd43c275ed9, entries=150, sequenceid=299, filesize=12.0 K 2024-12-03T05:01:21,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/602a06f9ccbf40908ddbddaebf21731f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/602a06f9ccbf40908ddbddaebf21731f 2024-12-03T05:01:21,989 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#A#compaction#349 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:21,990 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/25ea2924dd5e49869d99fa71c3eb37cc is 175, key is test_row_0/A:col10/1733202079764/Put/seqid=0 2024-12-03T05:01:22,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/602a06f9ccbf40908ddbddaebf21731f, entries=150, sequenceid=299, filesize=12.0 K 2024-12-03T05:01:22,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for eb947ae323132c3b2ada531b3598951f in 273ms, sequenceid=299, compaction requested=false 2024-12-03T05:01:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:22,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742251_1427 (size=31937) 2024-12-03T05:01:22,010 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/25ea2924dd5e49869d99fa71c3eb37cc as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/25ea2924dd5e49869d99fa71c3eb37cc 2024-12-03T05:01:22,031 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/A of eb947ae323132c3b2ada531b3598951f into 25ea2924dd5e49869d99fa71c3eb37cc(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:22,031 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:22,031 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/A, priority=13, startTime=1733202081475; duration=0sec 2024-12-03T05:01:22,031 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:22,031 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:A 2024-12-03T05:01:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:01:22,071 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:22,071 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-03T05:01:22,074 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:01:22,075 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:22,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:22,108 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/698463f3862e45bbae4d32ab5ee023a1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/698463f3862e45bbae4d32ab5ee023a1 2024-12-03T05:01:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:22,116 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:22,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:01:22,123 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/C of eb947ae323132c3b2ada531b3598951f into 698463f3862e45bbae4d32ab5ee023a1(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:22,123 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:22,123 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/C, priority=13, startTime=1733202081476; duration=0sec 2024-12-03T05:01:22,123 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:22,123 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:C 2024-12-03T05:01:22,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038abafee465ab4e3bb5d66e0f691a472f_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202081750/Put/seqid=0 2024-12-03T05:01:22,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742252_1428 (size=14994) 2024-12-03T05:01:22,166 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:22,177 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038abafee465ab4e3bb5d66e0f691a472f_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038abafee465ab4e3bb5d66e0f691a472f_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:01:22,184 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6b40519853554b85a3b1954096dceefc, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:22,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6b40519853554b85a3b1954096dceefc is 175, key is test_row_0/A:col10/1733202081750/Put/seqid=0 2024-12-03T05:01:22,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742253_1429 (size=39949) 2024-12-03T05:01:22,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202142227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202142229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T05:01:22,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:22,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202142234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202142244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202142244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202142363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202142363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202142363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202142363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202142370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:01:22,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T05:01:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T05:01:22,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:22,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202142581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202142581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202142581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202142581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202142582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,624 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6b40519853554b85a3b1954096dceefc 2024-12-03T05:01:22,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/c62fcd0929014120aaa7c5e89d8a6bfa is 50, key is test_row_0/B:col10/1733202081750/Put/seqid=0 2024-12-03T05:01:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:01:22,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742254_1430 (size=12301) 2024-12-03T05:01:22,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/c62fcd0929014120aaa7c5e89d8a6bfa 2024-12-03T05:01:22,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T05:01:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/7287642efb08406aafcfad60d48e8d7c is 50, key is test_row_0/C:col10/1733202081750/Put/seqid=0 2024-12-03T05:01:22,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742255_1431 (size=12301) 2024-12-03T05:01:22,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/7287642efb08406aafcfad60d48e8d7c 2024-12-03T05:01:22,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/6b40519853554b85a3b1954096dceefc as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6b40519853554b85a3b1954096dceefc 2024-12-03T05:01:22,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6b40519853554b85a3b1954096dceefc, entries=200, sequenceid=311, filesize=39.0 K 2024-12-03T05:01:22,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/c62fcd0929014120aaa7c5e89d8a6bfa as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c62fcd0929014120aaa7c5e89d8a6bfa 2024-12-03T05:01:22,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c62fcd0929014120aaa7c5e89d8a6bfa, entries=150, sequenceid=311, filesize=12.0 K 2024-12-03T05:01:22,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/7287642efb08406aafcfad60d48e8d7c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7287642efb08406aafcfad60d48e8d7c 2024-12-03T05:01:22,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T05:01:22,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:22,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34324 deadline: 1733202142901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34254 deadline: 1733202142905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34276 deadline: 1733202142905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34250 deadline: 1733202142911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34308 deadline: 1733202142913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:01:22,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7287642efb08406aafcfad60d48e8d7c, entries=150, sequenceid=311, filesize=12.0 K 2024-12-03T05:01:22,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for eb947ae323132c3b2ada531b3598951f in 813ms, sequenceid=311, compaction requested=true 2024-12-03T05:01:22,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:22,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:22,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:22,929 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:22,930 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:22,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:22,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:22,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb947ae323132c3b2ada531b3598951f:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:22,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:22,937 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:22,937 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:22,937 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/B is initiating minor compaction (all files) 2024-12-03T05:01:22,937 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/A is initiating minor compaction (all files) 2024-12-03T05:01:22,938 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/B in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,938 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/A in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,938 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/ee0d2d1ff05449caa9f63500e472800d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/50d422317e964cd28ad7ecd43c275ed9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c62fcd0929014120aaa7c5e89d8a6bfa] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=36.7 K 2024-12-03T05:01:22,938 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/25ea2924dd5e49869d99fa71c3eb37cc, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/cbacd21f184441bc9182a4a77293c4f1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6b40519853554b85a3b1954096dceefc] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=109.2 K 2024-12-03T05:01:22,938 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:22,938 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/25ea2924dd5e49869d99fa71c3eb37cc, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/cbacd21f184441bc9182a4a77293c4f1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6b40519853554b85a3b1954096dceefc] 2024-12-03T05:01:22,940 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ee0d2d1ff05449caa9f63500e472800d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733202079744 2024-12-03T05:01:22,940 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25ea2924dd5e49869d99fa71c3eb37cc, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733202079744 2024-12-03T05:01:22,941 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d422317e964cd28ad7ecd43c275ed9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733202080513 2024-12-03T05:01:22,941 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbacd21f184441bc9182a4a77293c4f1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733202080513 2024-12-03T05:01:22,941 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c62fcd0929014120aaa7c5e89d8a6bfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733202081750 2024-12-03T05:01:22,941 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b40519853554b85a3b1954096dceefc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733202081739 2024-12-03T05:01:22,954 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:22,961 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#B#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:22,962 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/c2cc71c29e2746f1aa2549f132ee9b2b is 50, key is test_row_0/B:col10/1733202081750/Put/seqid=0 2024-12-03T05:01:22,966 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412036e7d575413c44d45b9ca981623e2c6b2_eb947ae323132c3b2ada531b3598951f store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:22,971 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412036e7d575413c44d45b9ca981623e2c6b2_eb947ae323132c3b2ada531b3598951f, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:22,971 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036e7d575413c44d45b9ca981623e2c6b2_eb947ae323132c3b2ada531b3598951f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:22,986 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-12-03T05:01:22,986 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:22,986 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:22,986 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:22,989 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-12-03T05:01:22,989 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:22,989 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:22,989 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:22,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:22,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:22,994 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-12-03T05:01:22,994 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:22,994 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:22,994 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:22,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:22,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742256_1432 (size=13085) 2024-12-03T05:01:22,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742257_1433 (size=4469) 2024-12-03T05:01:22,998 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-12-03T05:01:22,998 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:22,998 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:22,998 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:22,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:22,999 INFO [Thread-1552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1552. 2024-12-03T05:01:22,999 DEBUG [Thread-1552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:22,999 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:22,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:22,999 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:23,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T05:01:23,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:23,053 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-03T05:01:23,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203efad44440fe74a5abe4534475100a003_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202082238/Put/seqid=0 2024-12-03T05:01:23,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742258_1434 (size=12454) 2024-12-03T05:01:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:01:23,397 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/c2cc71c29e2746f1aa2549f132ee9b2b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c2cc71c29e2746f1aa2549f132ee9b2b 2024-12-03T05:01:23,398 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#A#compaction#357 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:23,399 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/2a0e4506df7748b2a0e91ec35dba5066 is 175, key is test_row_0/A:col10/1733202081750/Put/seqid=0 2024-12-03T05:01:23,402 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/B of eb947ae323132c3b2ada531b3598951f into c2cc71c29e2746f1aa2549f132ee9b2b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:23,402 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:23,402 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/B, priority=13, startTime=1733202082929; duration=0sec 2024-12-03T05:01:23,402 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:23,402 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:B 2024-12-03T05:01:23,402 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:23,403 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:23,403 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): eb947ae323132c3b2ada531b3598951f/C is initiating minor compaction (all files) 2024-12-03T05:01:23,403 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb947ae323132c3b2ada531b3598951f/C in TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:23,403 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/698463f3862e45bbae4d32ab5ee023a1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/602a06f9ccbf40908ddbddaebf21731f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7287642efb08406aafcfad60d48e8d7c] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp, totalSize=36.7 K 2024-12-03T05:01:23,404 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 698463f3862e45bbae4d32ab5ee023a1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733202079744 2024-12-03T05:01:23,404 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 602a06f9ccbf40908ddbddaebf21731f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733202080513 2024-12-03T05:01:23,404 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7287642efb08406aafcfad60d48e8d7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733202081750 2024-12-03T05:01:23,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742259_1435 (size=32039) 2024-12-03T05:01:23,413 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb947ae323132c3b2ada531b3598951f#C#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:23,413 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/3ef8525f69e94b109a5afd0c50690be6 is 50, key is test_row_0/C:col10/1733202081750/Put/seqid=0 2024-12-03T05:01:23,414 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/2a0e4506df7748b2a0e91ec35dba5066 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/2a0e4506df7748b2a0e91ec35dba5066 2024-12-03T05:01:23,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742260_1436 (size=13085) 2024-12-03T05:01:23,418 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/A of eb947ae323132c3b2ada531b3598951f into 2a0e4506df7748b2a0e91ec35dba5066(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:23,418 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:23,418 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/A, priority=13, startTime=1733202082929; duration=0sec 2024-12-03T05:01:23,419 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:23,419 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:A 2024-12-03T05:01:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:23,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. as already flushing 2024-12-03T05:01:23,422 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-12-03T05:01:23,422 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:23,422 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:23,422 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:23,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:23,425 INFO [Thread-1545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1545. 2024-12-03T05:01:23,425 DEBUG [Thread-1545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:23,425 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:23,425 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:23,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:23,434 INFO [Thread-1547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1547. 2024-12-03T05:01:23,434 DEBUG [Thread-1547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:23,434 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:23,434 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:23,434 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:23,434 INFO [Thread-1541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1541. 2024-12-03T05:01:23,435 DEBUG [Thread-1541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:23,435 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:23,435 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:23,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:23,436 INFO [Thread-1543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1543. 2024-12-03T05:01:23,436 DEBUG [Thread-1543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:23,436 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:23,436 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:23,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:23,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:23,466 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203efad44440fe74a5abe4534475100a003_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203efad44440fe74a5abe4534475100a003_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:23,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ae0801215e8944b8bf159892d794998a, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:23,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ae0801215e8944b8bf159892d794998a is 175, key is test_row_0/A:col10/1733202082238/Put/seqid=0 2024-12-03T05:01:23,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742261_1437 (size=31255) 2024-12-03T05:01:23,474 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ae0801215e8944b8bf159892d794998a 2024-12-03T05:01:23,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/993f1fe679134930a02e95c8de17cf7c is 50, key is test_row_0/B:col10/1733202082238/Put/seqid=0 2024-12-03T05:01:23,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742262_1438 (size=12301) 2024-12-03T05:01:23,822 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/3ef8525f69e94b109a5afd0c50690be6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/3ef8525f69e94b109a5afd0c50690be6 2024-12-03T05:01:23,825 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb947ae323132c3b2ada531b3598951f/C of eb947ae323132c3b2ada531b3598951f into 3ef8525f69e94b109a5afd0c50690be6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:23,825 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:23,825 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., storeName=eb947ae323132c3b2ada531b3598951f/C, priority=13, startTime=1733202082930; duration=0sec 2024-12-03T05:01:23,826 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:23,826 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb947ae323132c3b2ada531b3598951f:C 2024-12-03T05:01:23,885 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/993f1fe679134930a02e95c8de17cf7c 2024-12-03T05:01:23,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/8ce7991534c24eb892d783d2b11bac23 is 50, key is test_row_0/C:col10/1733202082238/Put/seqid=0 2024-12-03T05:01:23,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742263_1439 (size=12301) 2024-12-03T05:01:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:01:24,294 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/8ce7991534c24eb892d783d2b11bac23 2024-12-03T05:01:24,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/ae0801215e8944b8bf159892d794998a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ae0801215e8944b8bf159892d794998a 2024-12-03T05:01:24,301 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ae0801215e8944b8bf159892d794998a, entries=150, sequenceid=338, filesize=30.5 K 2024-12-03T05:01:24,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/993f1fe679134930a02e95c8de17cf7c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/993f1fe679134930a02e95c8de17cf7c 2024-12-03T05:01:24,304 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/993f1fe679134930a02e95c8de17cf7c, entries=150, sequenceid=338, filesize=12.0 K 2024-12-03T05:01:24,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/8ce7991534c24eb892d783d2b11bac23 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/8ce7991534c24eb892d783d2b11bac23 2024-12-03T05:01:24,308 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/8ce7991534c24eb892d783d2b11bac23, entries=150, sequenceid=338, filesize=12.0 K 2024-12-03T05:01:24,308 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=33.54 KB/34350 for eb947ae323132c3b2ada531b3598951f in 1255ms, sequenceid=338, compaction requested=false 2024-12-03T05:01:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for eb947ae323132c3b2ada531b3598951f: 2024-12-03T05:01:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-03T05:01:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-03T05:01:24,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-03T05:01:24,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2350 sec 2024-12-03T05:01:24,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.2410 sec 2024-12-03T05:01:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:01:26,241 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:26,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T05:01:26,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-12-03T05:01:26,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-03T05:01:26,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-12-03T05:01:26,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-12-03T05:01:26,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-12-03T05:01:26,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T05:01:26,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T05:01:26,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 952 2024-12-03T05:01:26,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2856 rows 2024-12-03T05:01:26,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 954 2024-12-03T05:01:26,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2861 rows 2024-12-03T05:01:26,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 947 2024-12-03T05:01:26,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2841 rows 2024-12-03T05:01:26,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 943 2024-12-03T05:01:26,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2829 rows 2024-12-03T05:01:26,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 946 2024-12-03T05:01:26,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2836 rows 2024-12-03T05:01:26,242 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:01:26,242 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:01:26,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:26,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:26,242 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:26,243 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:01:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:01:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:01:26,246 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202086246"}]},"ts":"1733202086246"} 2024-12-03T05:01:26,248 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:01:26,248 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:01:26,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:01:26,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, UNASSIGN}] 2024-12-03T05:01:26,251 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, UNASSIGN 2024-12-03T05:01:26,251 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=eb947ae323132c3b2ada531b3598951f, regionState=CLOSING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:01:26,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, UNASSIGN because future has completed 2024-12-03T05:01:26,253 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:01:26,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:01:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:01:26,405 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:26,405 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:01:26,405 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing eb947ae323132c3b2ada531b3598951f, disabling compactions & flushes 2024-12-03T05:01:26,405 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:26,405 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:26,405 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. after waiting 0 ms 2024-12-03T05:01:26,405 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:26,405 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing eb947ae323132c3b2ada531b3598951f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-03T05:01:26,410 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e68d439f9f4942e692b02e51661f07e9_eb947ae323132c3b2ada531b3598951f is 50, key is test_row_0/A:col10/1733202083431/Put/seqid=0 2024-12-03T05:01:26,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742264_1440 (size=12454) 2024-12-03T05:01:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:01:26,814 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:26,818 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e68d439f9f4942e692b02e51661f07e9_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e68d439f9f4942e692b02e51661f07e9_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:26,819 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/69d92f256ff9409bbcf54e048de9c646, store: [table=TestAcidGuarantees family=A region=eb947ae323132c3b2ada531b3598951f] 2024-12-03T05:01:26,820 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/69d92f256ff9409bbcf54e048de9c646 is 175, key is test_row_0/A:col10/1733202083431/Put/seqid=0 2024-12-03T05:01:26,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742265_1441 (size=31255) 2024-12-03T05:01:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:01:27,224 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/69d92f256ff9409bbcf54e048de9c646 2024-12-03T05:01:27,231 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/9fea4416f38c4beab86a5f326cc74917 is 50, key is test_row_0/B:col10/1733202083431/Put/seqid=0 2024-12-03T05:01:27,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742266_1442 (size=12301) 2024-12-03T05:01:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:01:27,635 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/9fea4416f38c4beab86a5f326cc74917 2024-12-03T05:01:27,641 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/517c4fd96ec346629d165d853c619b8a is 50, key is test_row_0/C:col10/1733202083431/Put/seqid=0 2024-12-03T05:01:27,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742267_1443 (size=12301) 2024-12-03T05:01:28,045 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/517c4fd96ec346629d165d853c619b8a 2024-12-03T05:01:28,049 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/A/69d92f256ff9409bbcf54e048de9c646 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/69d92f256ff9409bbcf54e048de9c646 2024-12-03T05:01:28,052 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/69d92f256ff9409bbcf54e048de9c646, entries=150, sequenceid=349, filesize=30.5 K 2024-12-03T05:01:28,053 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/B/9fea4416f38c4beab86a5f326cc74917 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/9fea4416f38c4beab86a5f326cc74917 2024-12-03T05:01:28,056 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/9fea4416f38c4beab86a5f326cc74917, entries=150, sequenceid=349, filesize=12.0 K 2024-12-03T05:01:28,057 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/.tmp/C/517c4fd96ec346629d165d853c619b8a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/517c4fd96ec346629d165d853c619b8a 2024-12-03T05:01:28,060 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/517c4fd96ec346629d165d853c619b8a, entries=150, sequenceid=349, filesize=12.0 K 2024-12-03T05:01:28,060 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for eb947ae323132c3b2ada531b3598951f in 1655ms, sequenceid=349, compaction requested=true 2024-12-03T05:01:28,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6f9310e2f5ec41b4a214c1aab8a36cd6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f3f48b6d925a4c2f9d0d23952a180df1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ba2c518d008b4b8b8880b08eda462702, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0504731b5ad644cfb2e13135b273518e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/975e3834bcb146beb6ea6a33d5ac91ca, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/750cec1724f8483cae35e51fc5bad4ff, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0bc5cce5f26748b9935e13a566ba8107, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/91588a1669f646948d625dd6c2aa2166, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ac605d7b23d44e2abe759adc43a0d79d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/2d40230fe56e497ab298b0139390e822, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/029e25832c6a4432a78d33e83c305593, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8b2b2b0c964473b99d5c632d819e6af, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/564f132191ef4cfda8ad800e3da6a4a5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f63e04d6b8714a58bf6fe4a4fb4f7c5f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/70ea5d323c6f4334b7752a7a5979ba81, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8ea881d89d44f2ca8bf0163b8dc1acd, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ca304a4b35914746835e9140e0fb37a8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/e217d26e4ea84e248a78d362d70e2341, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0ffca7d28c6a4fea95793d797ecfa4f5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6d95cdb3135944ecbd20a5ff49174da9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/25ea2924dd5e49869d99fa71c3eb37cc, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/cbacd21f184441bc9182a4a77293c4f1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6b40519853554b85a3b1954096dceefc] to archive 2024-12-03T05:01:28,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:01:28,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6f9310e2f5ec41b4a214c1aab8a36cd6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6f9310e2f5ec41b4a214c1aab8a36cd6 2024-12-03T05:01:28,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f3f48b6d925a4c2f9d0d23952a180df1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f3f48b6d925a4c2f9d0d23952a180df1 2024-12-03T05:01:28,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ba2c518d008b4b8b8880b08eda462702 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ba2c518d008b4b8b8880b08eda462702 2024-12-03T05:01:28,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0504731b5ad644cfb2e13135b273518e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0504731b5ad644cfb2e13135b273518e 2024-12-03T05:01:28,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/975e3834bcb146beb6ea6a33d5ac91ca to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/975e3834bcb146beb6ea6a33d5ac91ca 2024-12-03T05:01:28,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/750cec1724f8483cae35e51fc5bad4ff to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/750cec1724f8483cae35e51fc5bad4ff 2024-12-03T05:01:28,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0bc5cce5f26748b9935e13a566ba8107 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0bc5cce5f26748b9935e13a566ba8107 2024-12-03T05:01:28,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/91588a1669f646948d625dd6c2aa2166 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/91588a1669f646948d625dd6c2aa2166 2024-12-03T05:01:28,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ac605d7b23d44e2abe759adc43a0d79d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ac605d7b23d44e2abe759adc43a0d79d 2024-12-03T05:01:28,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/2d40230fe56e497ab298b0139390e822 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/2d40230fe56e497ab298b0139390e822 2024-12-03T05:01:28,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/029e25832c6a4432a78d33e83c305593 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/029e25832c6a4432a78d33e83c305593 2024-12-03T05:01:28,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8b2b2b0c964473b99d5c632d819e6af to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8b2b2b0c964473b99d5c632d819e6af 2024-12-03T05:01:28,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/564f132191ef4cfda8ad800e3da6a4a5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/564f132191ef4cfda8ad800e3da6a4a5 2024-12-03T05:01:28,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f63e04d6b8714a58bf6fe4a4fb4f7c5f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f63e04d6b8714a58bf6fe4a4fb4f7c5f 2024-12-03T05:01:28,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/70ea5d323c6f4334b7752a7a5979ba81 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/70ea5d323c6f4334b7752a7a5979ba81 2024-12-03T05:01:28,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8ea881d89d44f2ca8bf0163b8dc1acd to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/f8ea881d89d44f2ca8bf0163b8dc1acd 2024-12-03T05:01:28,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ca304a4b35914746835e9140e0fb37a8 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ca304a4b35914746835e9140e0fb37a8 2024-12-03T05:01:28,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/e217d26e4ea84e248a78d362d70e2341 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/e217d26e4ea84e248a78d362d70e2341 2024-12-03T05:01:28,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0ffca7d28c6a4fea95793d797ecfa4f5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/0ffca7d28c6a4fea95793d797ecfa4f5 2024-12-03T05:01:28,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6d95cdb3135944ecbd20a5ff49174da9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6d95cdb3135944ecbd20a5ff49174da9 2024-12-03T05:01:28,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/25ea2924dd5e49869d99fa71c3eb37cc to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/25ea2924dd5e49869d99fa71c3eb37cc 2024-12-03T05:01:28,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/cbacd21f184441bc9182a4a77293c4f1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/cbacd21f184441bc9182a4a77293c4f1 2024-12-03T05:01:28,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6b40519853554b85a3b1954096dceefc to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/6b40519853554b85a3b1954096dceefc 2024-12-03T05:01:28,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/8b736627260a47d5a005225c5f205bba, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/499272f33d6c4186b2cf6ae4419fc873, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/d8518eae2215424f927e70eb72d3cd76, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/5ba42c78c2934d2281b6d1c9016b3fbc, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/769524cf267847b8b454ba933e072eed, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/872681070bf64aee92e5141aafef36b1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/39dd52b3935d4f59a01bf2c9b6e80d54, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/593e2067c1854c2e94f276ae48f41670, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/0096bba05b764fb0ae8737edd4efc717, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c1ee555076ca432f97affc36195e5dc5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/30a318763ca2413c95c8b1179035b999, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/022a765a484c4ed0ad0c860317f873b3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/85cd35b5341942c79348d6edeb9c8ee2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/6a4f04f6e4534edf9c9681746f515da0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a81d296d05e940f9967014e06d65dae1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/1c4394bf57a04bbbaeba3a2e85e07fc3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/dfc68c54d30349eea7219d16218cf990, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/f177649326cb4697b1defd9bb0511b54, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a372f7e05abf43688857a9f242a5aef2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/ee0d2d1ff05449caa9f63500e472800d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/80355a2b86754981800f60a9de729aad, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/50d422317e964cd28ad7ecd43c275ed9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c62fcd0929014120aaa7c5e89d8a6bfa] to archive 2024-12-03T05:01:28,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:01:28,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/8b736627260a47d5a005225c5f205bba to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/8b736627260a47d5a005225c5f205bba 2024-12-03T05:01:28,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/499272f33d6c4186b2cf6ae4419fc873 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/499272f33d6c4186b2cf6ae4419fc873 2024-12-03T05:01:28,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/d8518eae2215424f927e70eb72d3cd76 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/d8518eae2215424f927e70eb72d3cd76 2024-12-03T05:01:28,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/5ba42c78c2934d2281b6d1c9016b3fbc to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/5ba42c78c2934d2281b6d1c9016b3fbc 2024-12-03T05:01:28,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/769524cf267847b8b454ba933e072eed to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/769524cf267847b8b454ba933e072eed 2024-12-03T05:01:28,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/872681070bf64aee92e5141aafef36b1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/872681070bf64aee92e5141aafef36b1 2024-12-03T05:01:28,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/39dd52b3935d4f59a01bf2c9b6e80d54 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/39dd52b3935d4f59a01bf2c9b6e80d54 2024-12-03T05:01:28,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/593e2067c1854c2e94f276ae48f41670 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/593e2067c1854c2e94f276ae48f41670 2024-12-03T05:01:28,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/0096bba05b764fb0ae8737edd4efc717 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/0096bba05b764fb0ae8737edd4efc717 2024-12-03T05:01:28,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c1ee555076ca432f97affc36195e5dc5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c1ee555076ca432f97affc36195e5dc5 2024-12-03T05:01:28,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/30a318763ca2413c95c8b1179035b999 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/30a318763ca2413c95c8b1179035b999 2024-12-03T05:01:28,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/022a765a484c4ed0ad0c860317f873b3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/022a765a484c4ed0ad0c860317f873b3 2024-12-03T05:01:28,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/85cd35b5341942c79348d6edeb9c8ee2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/85cd35b5341942c79348d6edeb9c8ee2 2024-12-03T05:01:28,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/6a4f04f6e4534edf9c9681746f515da0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/6a4f04f6e4534edf9c9681746f515da0 2024-12-03T05:01:28,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a81d296d05e940f9967014e06d65dae1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a81d296d05e940f9967014e06d65dae1 2024-12-03T05:01:28,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/1c4394bf57a04bbbaeba3a2e85e07fc3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/1c4394bf57a04bbbaeba3a2e85e07fc3 2024-12-03T05:01:28,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/dfc68c54d30349eea7219d16218cf990 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/dfc68c54d30349eea7219d16218cf990 2024-12-03T05:01:28,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/f177649326cb4697b1defd9bb0511b54 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/f177649326cb4697b1defd9bb0511b54 2024-12-03T05:01:28,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a372f7e05abf43688857a9f242a5aef2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/a372f7e05abf43688857a9f242a5aef2 2024-12-03T05:01:28,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/ee0d2d1ff05449caa9f63500e472800d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/ee0d2d1ff05449caa9f63500e472800d 2024-12-03T05:01:28,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/80355a2b86754981800f60a9de729aad to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/80355a2b86754981800f60a9de729aad 2024-12-03T05:01:28,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/50d422317e964cd28ad7ecd43c275ed9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/50d422317e964cd28ad7ecd43c275ed9 2024-12-03T05:01:28,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c62fcd0929014120aaa7c5e89d8a6bfa to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c62fcd0929014120aaa7c5e89d8a6bfa 2024-12-03T05:01:28,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/59feb42b1f5f4cf89eb7ff933d5d90a8, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/f9029180db13485b9315bf0ffb4f134f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/9f72847217884772b2b7d94cb9e050ba, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/c8ab83cca4a24f7398dd2eb645932fef, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ffa9e0b51aed4427a188d608a27d8938, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/e409b64a3ae6453d817a0d43c881483a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/0abf964982a8480994dbad3c6a17036c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/1171d44197124b97b7d35723b87338da, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/97696ace5ffa4ea0ba5358fba8c52a4f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/2f485684c74c4e1ab2caf5dc241ea46a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/93a8d5c5d6914273854cc8ce7720e3d0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7a2e80f758644ad0ad6a3e6796c8d9b7, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/02f4d5c011c84e3ba9f9507d9b4d07bb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/c4811770c4dc44b59bab1c6b8634a5be, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/a1b631f0168e4b12830a07580a78f74c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ff0899071c0a430d8fa029fc87228a19, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/3adfafa55e17423eb663dca97b89e406, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/aead914ca7504657a6feb96b4258af58, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/280428ab73534df6951498ab75465cdb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/698463f3862e45bbae4d32ab5ee023a1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/609776b0b76140b4b921ea87cec40a22, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/602a06f9ccbf40908ddbddaebf21731f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7287642efb08406aafcfad60d48e8d7c] to archive 2024-12-03T05:01:28,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:01:28,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/59feb42b1f5f4cf89eb7ff933d5d90a8 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/59feb42b1f5f4cf89eb7ff933d5d90a8 2024-12-03T05:01:28,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/f9029180db13485b9315bf0ffb4f134f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/f9029180db13485b9315bf0ffb4f134f 2024-12-03T05:01:28,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/9f72847217884772b2b7d94cb9e050ba to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/9f72847217884772b2b7d94cb9e050ba 2024-12-03T05:01:28,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/c8ab83cca4a24f7398dd2eb645932fef to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/c8ab83cca4a24f7398dd2eb645932fef 2024-12-03T05:01:28,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ffa9e0b51aed4427a188d608a27d8938 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ffa9e0b51aed4427a188d608a27d8938 2024-12-03T05:01:28,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/e409b64a3ae6453d817a0d43c881483a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/e409b64a3ae6453d817a0d43c881483a 2024-12-03T05:01:28,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/0abf964982a8480994dbad3c6a17036c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/0abf964982a8480994dbad3c6a17036c 2024-12-03T05:01:28,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/1171d44197124b97b7d35723b87338da to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/1171d44197124b97b7d35723b87338da 2024-12-03T05:01:28,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/97696ace5ffa4ea0ba5358fba8c52a4f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/97696ace5ffa4ea0ba5358fba8c52a4f 2024-12-03T05:01:28,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/2f485684c74c4e1ab2caf5dc241ea46a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/2f485684c74c4e1ab2caf5dc241ea46a 2024-12-03T05:01:28,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/93a8d5c5d6914273854cc8ce7720e3d0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/93a8d5c5d6914273854cc8ce7720e3d0 2024-12-03T05:01:28,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7a2e80f758644ad0ad6a3e6796c8d9b7 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7a2e80f758644ad0ad6a3e6796c8d9b7 2024-12-03T05:01:28,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/02f4d5c011c84e3ba9f9507d9b4d07bb to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/02f4d5c011c84e3ba9f9507d9b4d07bb 2024-12-03T05:01:28,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/c4811770c4dc44b59bab1c6b8634a5be to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/c4811770c4dc44b59bab1c6b8634a5be 2024-12-03T05:01:28,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/a1b631f0168e4b12830a07580a78f74c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/a1b631f0168e4b12830a07580a78f74c 2024-12-03T05:01:28,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ff0899071c0a430d8fa029fc87228a19 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/ff0899071c0a430d8fa029fc87228a19 2024-12-03T05:01:28,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/3adfafa55e17423eb663dca97b89e406 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/3adfafa55e17423eb663dca97b89e406 2024-12-03T05:01:28,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/aead914ca7504657a6feb96b4258af58 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/aead914ca7504657a6feb96b4258af58 2024-12-03T05:01:28,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/280428ab73534df6951498ab75465cdb to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/280428ab73534df6951498ab75465cdb 2024-12-03T05:01:28,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/698463f3862e45bbae4d32ab5ee023a1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/698463f3862e45bbae4d32ab5ee023a1 2024-12-03T05:01:28,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/609776b0b76140b4b921ea87cec40a22 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/609776b0b76140b4b921ea87cec40a22 2024-12-03T05:01:28,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/602a06f9ccbf40908ddbddaebf21731f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/602a06f9ccbf40908ddbddaebf21731f 2024-12-03T05:01:28,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7287642efb08406aafcfad60d48e8d7c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/7287642efb08406aafcfad60d48e8d7c 2024-12-03T05:01:28,146 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/recovered.edits/352.seqid, newMaxSeqId=352, maxSeqId=4 2024-12-03T05:01:28,147 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. 2024-12-03T05:01:28,147 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for eb947ae323132c3b2ada531b3598951f: Waiting for close lock at 1733202086405Running coprocessor pre-close hooks at 1733202086405Disabling compacts and flushes for region at 1733202086405Disabling writes for close at 1733202086405Obtaining lock to block concurrent updates at 1733202086405Preparing flush snapshotting stores in eb947ae323132c3b2ada531b3598951f at 1733202086405Finished memstore snapshotting TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733202086405Flushing stores of TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f. at 1733202086406 (+1 ms)Flushing eb947ae323132c3b2ada531b3598951f/A: creating writer at 1733202086406Flushing mob file eb947ae323132c3b2ada531b3598951f/A: appending metadata at 1733202086410 (+4 ms)Flushing mob file eb947ae323132c3b2ada531b3598951f/A: closing flushed file at 1733202086410Flushing eb947ae323132c3b2ada531b3598951f/A: appending metadata at 1733202086819 (+409 ms)Flushing eb947ae323132c3b2ada531b3598951f/A: closing flushed file at 1733202086819Flushing eb947ae323132c3b2ada531b3598951f/B: creating writer at 1733202087228 (+409 ms)Flushing eb947ae323132c3b2ada531b3598951f/B: appending metadata at 1733202087230 (+2 ms)Flushing eb947ae323132c3b2ada531b3598951f/B: closing flushed file at 1733202087231 (+1 ms)Flushing eb947ae323132c3b2ada531b3598951f/C: creating writer at 1733202087639 (+408 ms)Flushing eb947ae323132c3b2ada531b3598951f/C: appending metadata at 1733202087641 (+2 ms)Flushing eb947ae323132c3b2ada531b3598951f/C: closing flushed file at 1733202087641Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67355b2e: reopening flushed file at 1733202088048 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@300ab3cb: reopening flushed file at 1733202088052 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f7cbf92: reopening flushed file at 1733202088056 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for eb947ae323132c3b2ada531b3598951f in 1655ms, sequenceid=349, compaction requested=true at 1733202088060 (+4 ms)Writing region close event to WAL at 1733202088144 (+84 ms)Running coprocessor post-close hooks at 1733202088147 (+3 ms)Closed at 1733202088147 2024-12-03T05:01:28,149 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,150 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=eb947ae323132c3b2ada531b3598951f, regionState=CLOSED 2024-12-03T05:01:28,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:01:28,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-03T05:01:28,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure eb947ae323132c3b2ada531b3598951f, server=32beccd76b35,34719,1733201977102 in 1.8990 sec 2024-12-03T05:01:28,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-03T05:01:28,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb947ae323132c3b2ada531b3598951f, UNASSIGN in 1.9030 sec 2024-12-03T05:01:28,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-03T05:01:28,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9060 sec 2024-12-03T05:01:28,158 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202088158"}]},"ts":"1733202088158"} 2024-12-03T05:01:28,159 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T05:01:28,159 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T05:01:28,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9160 sec 2024-12-03T05:01:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:01:28,401 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:28,401 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T05:01:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:01:28,403 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:01:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T05:01:28,404 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:01:28,406 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,408 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/recovered.edits] 2024-12-03T05:01:28,412 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/2a0e4506df7748b2a0e91ec35dba5066 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/2a0e4506df7748b2a0e91ec35dba5066 2024-12-03T05:01:28,413 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/69d92f256ff9409bbcf54e048de9c646 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/69d92f256ff9409bbcf54e048de9c646 2024-12-03T05:01:28,417 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ae0801215e8944b8bf159892d794998a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/A/ae0801215e8944b8bf159892d794998a 2024-12-03T05:01:28,421 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/993f1fe679134930a02e95c8de17cf7c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/993f1fe679134930a02e95c8de17cf7c 2024-12-03T05:01:28,422 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/9fea4416f38c4beab86a5f326cc74917 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/9fea4416f38c4beab86a5f326cc74917 2024-12-03T05:01:28,426 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c2cc71c29e2746f1aa2549f132ee9b2b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/B/c2cc71c29e2746f1aa2549f132ee9b2b 2024-12-03T05:01:28,428 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/3ef8525f69e94b109a5afd0c50690be6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/3ef8525f69e94b109a5afd0c50690be6 2024-12-03T05:01:28,429 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/517c4fd96ec346629d165d853c619b8a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/517c4fd96ec346629d165d853c619b8a 2024-12-03T05:01:28,430 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/8ce7991534c24eb892d783d2b11bac23 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/C/8ce7991534c24eb892d783d2b11bac23 2024-12-03T05:01:28,433 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/recovered.edits/352.seqid to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f/recovered.edits/352.seqid 2024-12-03T05:01:28,433 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,433 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T05:01:28,434 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T05:01:28,435 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T05:01:28,437 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203152b02bac6944d4a88328f3c298a7059_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203152b02bac6944d4a88328f3c298a7059_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,439 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032c49c334305743ee9224d0c75fe0f7e4_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032c49c334305743ee9224d0c75fe0f7e4_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,440 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203488ffc05a6584181916934db51b890f6_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203488ffc05a6584181916934db51b890f6_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,442 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120351b142fab61e4634bfdd578cd85a2d3a_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120351b142fab61e4634bfdd578cd85a2d3a_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,443 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035753131302f848d7b05a201e5a31323d_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035753131302f848d7b05a201e5a31323d_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,445 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203654db027beb446a0924b6e80b4c4a743_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203654db027beb446a0924b6e80b4c4a743_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,447 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203655fe06a1a2f4623be229d7491159949_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203655fe06a1a2f4623be229d7491159949_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,448 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203850590de449c4dd4825737b380730a60_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203850590de449c4dd4825737b380730a60_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,449 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203856a5270973f45f7b117829b9379f3ce_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203856a5270973f45f7b117829b9379f3ce_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,450 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389c636d2403c4e50b8191143048c0ff4_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389c636d2403c4e50b8191143048c0ff4_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,451 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038abafee465ab4e3bb5d66e0f691a472f_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038abafee465ab4e3bb5d66e0f691a472f_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,452 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038e43bf971fc448349da9848d0bc1885e_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038e43bf971fc448349da9848d0bc1885e_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,453 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039252cbfe2f6b447a9e2069fbd29962d8_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039252cbfe2f6b447a9e2069fbd29962d8_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,454 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b1676d110a754ed0ae6382464e9524e2_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b1676d110a754ed0ae6382464e9524e2_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,455 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c216f880ffee47d98936ebcfcd03b2f0_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c216f880ffee47d98936ebcfcd03b2f0_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,457 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d1844324b15f46d4a73bc37c7a5ae2cc_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d1844324b15f46d4a73bc37c7a5ae2cc_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,458 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e68d439f9f4942e692b02e51661f07e9_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e68d439f9f4942e692b02e51661f07e9_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,459 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203efad44440fe74a5abe4534475100a003_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203efad44440fe74a5abe4534475100a003_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,460 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f1d83c23e80547d8978a04df216f72d9_eb947ae323132c3b2ada531b3598951f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f1d83c23e80547d8978a04df216f72d9_eb947ae323132c3b2ada531b3598951f 2024-12-03T05:01:28,461 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T05:01:28,463 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:01:28,465 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T05:01:28,467 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T05:01:28,468 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:01:28,468 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T05:01:28,468 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733202088468"}]},"ts":"9223372036854775807"} 2024-12-03T05:01:28,469 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T05:01:28,469 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => eb947ae323132c3b2ada531b3598951f, NAME => 'TestAcidGuarantees,,1733202059779.eb947ae323132c3b2ada531b3598951f.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T05:01:28,469 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T05:01:28,470 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733202088469"}]},"ts":"9223372036854775807"} 2024-12-03T05:01:28,471 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T05:01:28,472 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:01:28,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-12-03T05:01:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T05:01:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T05:01:28,511 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:28,522 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 240) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 857), ProcessCount=11 (was 11), AvailableMemoryMB=3301 (was 3538) 2024-12-03T05:01:28,531 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=3301 2024-12-03T05:01:28,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:01:28,532 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:01:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T05:01:28,534 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:01:28,534 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:28,534 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-12-03T05:01:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:01:28,535 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:01:28,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742268_1444 (size=959) 2024-12-03T05:01:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:01:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:01:28,942 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d 2024-12-03T05:01:28,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742269_1445 (size=53) 2024-12-03T05:01:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:01:29,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:01:29,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1ef6c76f0cc0543400edbcde21d158fa, disabling compactions & flushes 2024-12-03T05:01:29,347 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:29,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:29,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. after waiting 0 ms 2024-12-03T05:01:29,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:29,348 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:29,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1ef6c76f0cc0543400edbcde21d158fa: Waiting for close lock at 1733202089347Disabling compacts and flushes for region at 1733202089347Disabling writes for close at 1733202089348 (+1 ms)Writing region close event to WAL at 1733202089348Closed at 1733202089348 2024-12-03T05:01:29,349 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:01:29,349 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733202089349"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733202089349"}]},"ts":"1733202089349"} 2024-12-03T05:01:29,351 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:01:29,352 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:01:29,352 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202089352"}]},"ts":"1733202089352"} 2024-12-03T05:01:29,353 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T05:01:29,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ef6c76f0cc0543400edbcde21d158fa, ASSIGN}] 2024-12-03T05:01:29,355 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ef6c76f0cc0543400edbcde21d158fa, ASSIGN 2024-12-03T05:01:29,355 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ef6c76f0cc0543400edbcde21d158fa, ASSIGN; state=OFFLINE, location=32beccd76b35,34719,1733201977102; forceNewPlan=false, retain=false 2024-12-03T05:01:29,506 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=1ef6c76f0cc0543400edbcde21d158fa, regionState=OPENING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:01:29,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ef6c76f0cc0543400edbcde21d158fa, ASSIGN because future has completed 2024-12-03T05:01:29,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:01:29,662 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:29,662 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:01:29,663 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,663 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:01:29,663 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,663 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,664 INFO [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,665 INFO [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef6c76f0cc0543400edbcde21d158fa columnFamilyName A 2024-12-03T05:01:29,665 DEBUG [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:29,666 INFO [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] regionserver.HStore(327): Store=1ef6c76f0cc0543400edbcde21d158fa/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:01:29,666 INFO [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,667 INFO [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef6c76f0cc0543400edbcde21d158fa columnFamilyName B 2024-12-03T05:01:29,667 DEBUG [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:29,667 INFO [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] regionserver.HStore(327): Store=1ef6c76f0cc0543400edbcde21d158fa/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:01:29,667 INFO [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,668 INFO [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef6c76f0cc0543400edbcde21d158fa columnFamilyName C 2024-12-03T05:01:29,668 DEBUG [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:01:29,668 INFO [StoreOpener-1ef6c76f0cc0543400edbcde21d158fa-1 {}] regionserver.HStore(327): Store=1ef6c76f0cc0543400edbcde21d158fa/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:01:29,669 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,669 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,669 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,670 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,670 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:01:29,670 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:01:29,671 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,672 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:01:29,673 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 1ef6c76f0cc0543400edbcde21d158fa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62954678, jitterRate=-0.06190219521522522}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:01:29,673 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:29,674 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 1ef6c76f0cc0543400edbcde21d158fa: Running coprocessor pre-open hook at 1733202089663Writing region info on filesystem at 1733202089663Initializing all the Stores at 1733202089664 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202089664Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202089664Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202089664Cleaning up temporary data from old regions at 1733202089670 (+6 ms)Running coprocessor post-open hooks at 1733202089673 (+3 ms)Region opened successfully at 1733202089673 2024-12-03T05:01:29,674 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., pid=116, masterSystemTime=1733202089660 2024-12-03T05:01:29,676 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:29,676 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:29,676 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=1ef6c76f0cc0543400edbcde21d158fa, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:01:29,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:01:29,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-03T05:01:29,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 in 170 msec 2024-12-03T05:01:29,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-03T05:01:29,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ef6c76f0cc0543400edbcde21d158fa, ASSIGN in 325 msec 2024-12-03T05:01:29,681 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:01:29,681 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202089681"}]},"ts":"1733202089681"} 2024-12-03T05:01:29,682 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T05:01:29,683 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:01:29,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-12-03T05:01:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:01:30,680 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:30,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:30,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:30,683 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c89f11d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:30,684 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179a3af6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,685 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:30,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:30,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:30,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:30,688 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:01:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,43423,1733201976038 2024-12-03T05:01:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2f6a7b00 2024-12-03T05:01:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:01:30,690 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:01:30,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:30,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:30,692 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477fa41b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:30,693 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@602fb477, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:30,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:30,695 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a38098f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:30,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:30,696 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:30,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dbd4437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:30,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:30,698 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25666f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:30,700 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5bdb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:30,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:30,702 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d0f71b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:30,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7915f995, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:30,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:30,706 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:30,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4afe0d57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:30,708 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b002ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:30,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:30,714 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af0ec9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:30,717 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e282d31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:30,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:30,719 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50b729c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:30,721 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51874cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:30,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:30,726 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747d71cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:30,729 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fce3353, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:30,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:30,732 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f69200e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:30,734 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf37d28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:01:30,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:01:30,738 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:01:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:01:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:01:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b78e41a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:01:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:01:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:30,740 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:01:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4440d96b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:01:30,741 DEBUG [Thread-1995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:30,741 DEBUG [Thread-2010 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:30,742 DEBUG [Thread-2006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:30,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:30,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:30,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:01:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:30,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:01:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:01:30,757 DEBUG [Thread-1997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:30,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:01:30,762 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:30,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:30,765 DEBUG [Thread-1999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:30,765 DEBUG [Thread-2012 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:01:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:30,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:30,769 DEBUG [Thread-1999 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:30,770 DEBUG [Thread-2004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:30,771 DEBUG [Thread-1999 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:30,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:01:30,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:01:30,779 DEBUG [Thread-2001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:30,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:30,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:01:30,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:01:30,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:30,787 DEBUG [Thread-2008 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:01:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:01:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:01:30,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:01:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-03T05:01:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2] 2024-12-03T05:01:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:01:30,793 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:30,794 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:30,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:30,895 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:30,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:01:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:01:30,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:01:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:30,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:30,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:30,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:30,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:30,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/e40e24f01fb64a44a2079a943bfdda18 is 50, key is test_row_0/A:col10/1733202090885/Put/seqid=0 2024-12-03T05:01:31,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742270_1446 (size=14341) 2024-12-03T05:01:31,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202151040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33850 deadline: 1733202151050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202151058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202151076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202151083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:01:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:01:31,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202151181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202151190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33850 deadline: 1733202151190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202151201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202151210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:01:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:31,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202151401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202151412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33850 deadline: 1733202151412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:01:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202151420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202151420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:01:31,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/e40e24f01fb64a44a2079a943bfdda18 2024-12-03T05:01:31,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/8a63039d8c4646fc8e9b45cbcd86b394 is 50, key is test_row_0/B:col10/1733202090885/Put/seqid=0 2024-12-03T05:01:31,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742271_1447 (size=12001) 2024-12-03T05:01:31,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/8a63039d8c4646fc8e9b45cbcd86b394 2024-12-03T05:01:31,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:01:31,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:31,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/efa5238fe5104c6dbb4c6b83d23dc774 is 50, key is test_row_0/C:col10/1733202090885/Put/seqid=0 2024-12-03T05:01:31,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742272_1448 (size=12001) 2024-12-03T05:01:31,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202151721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33850 deadline: 1733202151730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202151730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202151730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:01:31,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:31,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202151740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:31,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:01:31,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:31,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:31,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:31,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:01:32,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:01:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:32,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:32,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/efa5238fe5104c6dbb4c6b83d23dc774 2024-12-03T05:01:32,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/e40e24f01fb64a44a2079a943bfdda18 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e40e24f01fb64a44a2079a943bfdda18 2024-12-03T05:01:32,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e40e24f01fb64a44a2079a943bfdda18, entries=200, sequenceid=12, filesize=14.0 K 2024-12-03T05:01:32,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/8a63039d8c4646fc8e9b45cbcd86b394 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8a63039d8c4646fc8e9b45cbcd86b394 2024-12-03T05:01:32,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8a63039d8c4646fc8e9b45cbcd86b394, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T05:01:32,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/efa5238fe5104c6dbb4c6b83d23dc774 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/efa5238fe5104c6dbb4c6b83d23dc774 2024-12-03T05:01:32,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/efa5238fe5104c6dbb4c6b83d23dc774, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T05:01:32,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1ef6c76f0cc0543400edbcde21d158fa in 1219ms, sequenceid=12, compaction requested=false 2024-12-03T05:01:32,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:32,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:01:32,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:32,199 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-03T05:01:32,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/52ac4a71a17e41fc99e4e1dd9df44659 is 50, key is test_row_0/A:col10/1733202091073/Put/seqid=0 2024-12-03T05:01:32,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742273_1449 (size=12001) 2024-12-03T05:01:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:32,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:32,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33850 deadline: 1733202152250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202152251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202152255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202152256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202152260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202152370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202152370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202152370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202152580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202152580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202152580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,609 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/52ac4a71a17e41fc99e4e1dd9df44659 2024-12-03T05:01:32,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/97dd869da7fe4c04b1321e9ac0b9ebb9 is 50, key is test_row_0/B:col10/1733202091073/Put/seqid=0 2024-12-03T05:01:32,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742274_1450 (size=12001) 2024-12-03T05:01:32,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202152890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202152900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202152900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:01:33,020 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/97dd869da7fe4c04b1321e9ac0b9ebb9 2024-12-03T05:01:33,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/8aefb36cfd754f1890f1b12d9722e8d0 is 50, key is test_row_0/C:col10/1733202091073/Put/seqid=0 2024-12-03T05:01:33,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742275_1451 (size=12001) 2024-12-03T05:01:33,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:33,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33850 deadline: 1733202153270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:33,276 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_1' in 1ef6c76f0cc0543400edbcde21d158fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:33,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:33,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202153281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:33,287 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in 1ef6c76f0cc0543400edbcde21d158fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:33,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202153410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:33,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:33,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202153421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:33,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:33,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202153421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:33,443 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/8aefb36cfd754f1890f1b12d9722e8d0 2024-12-03T05:01:33,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/52ac4a71a17e41fc99e4e1dd9df44659 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/52ac4a71a17e41fc99e4e1dd9df44659 2024-12-03T05:01:33,457 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/52ac4a71a17e41fc99e4e1dd9df44659, entries=150, sequenceid=38, filesize=11.7 K 2024-12-03T05:01:33,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/97dd869da7fe4c04b1321e9ac0b9ebb9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/97dd869da7fe4c04b1321e9ac0b9ebb9 2024-12-03T05:01:33,470 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/97dd869da7fe4c04b1321e9ac0b9ebb9, entries=150, sequenceid=38, filesize=11.7 K 2024-12-03T05:01:33,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/8aefb36cfd754f1890f1b12d9722e8d0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/8aefb36cfd754f1890f1b12d9722e8d0 2024-12-03T05:01:33,483 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/8aefb36cfd754f1890f1b12d9722e8d0, entries=150, sequenceid=38, filesize=11.7 K 2024-12-03T05:01:33,485 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1ef6c76f0cc0543400edbcde21d158fa in 1286ms, sequenceid=38, compaction requested=false 2024-12-03T05:01:33,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:33,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:33,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-03T05:01:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-03T05:01:33,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-03T05:01:33,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6930 sec 2024-12-03T05:01:33,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.7040 sec 2024-12-03T05:01:33,678 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:01:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:34,436 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:34,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:01:34,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/3dfc6ca839304210837c03f8fd0c6b13 is 50, key is test_row_0/A:col10/1733202092253/Put/seqid=0 2024-12-03T05:01:34,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742276_1452 (size=14341) 2024-12-03T05:01:34,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/3dfc6ca839304210837c03f8fd0c6b13 2024-12-03T05:01:34,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/8ff4e1143926452f8354c96c48769088 is 50, key is test_row_0/B:col10/1733202092253/Put/seqid=0 2024-12-03T05:01:34,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742277_1453 (size=12001) 2024-12-03T05:01:34,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202154539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:34,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202154541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:34,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202154542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:34,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:34,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202154651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202154651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:34,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202154661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:34,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/8ff4e1143926452f8354c96c48769088 2024-12-03T05:01:34,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202154870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202154870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:34,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202154871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:34,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/351f266580f946059b3f5c591a56d302 is 50, key is test_row_0/C:col10/1733202092253/Put/seqid=0 2024-12-03T05:01:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:34,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742278_1454 (size=12001) 2024-12-03T05:01:34,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/351f266580f946059b3f5c591a56d302 2024-12-03T05:01:34,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/3dfc6ca839304210837c03f8fd0c6b13 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3dfc6ca839304210837c03f8fd0c6b13 2024-12-03T05:01:34,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3dfc6ca839304210837c03f8fd0c6b13, entries=200, sequenceid=49, filesize=14.0 K 2024-12-03T05:01:34,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/8ff4e1143926452f8354c96c48769088 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8ff4e1143926452f8354c96c48769088 2024-12-03T05:01:34,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8ff4e1143926452f8354c96c48769088, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T05:01:34,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/351f266580f946059b3f5c591a56d302 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/351f266580f946059b3f5c591a56d302 2024-12-03T05:01:34,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/351f266580f946059b3f5c591a56d302, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T05:01:34,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1ef6c76f0cc0543400edbcde21d158fa in 494ms, sequenceid=49, compaction requested=true 2024-12-03T05:01:34,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:34,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:34,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:34,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:34,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:34,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:34,931 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:34,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:01:34,931 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:34,932 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:01:34,933 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:34,933 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:34,933 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/A is initiating minor compaction (all files) 2024-12-03T05:01:34,933 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/B is initiating minor compaction (all files) 2024-12-03T05:01:34,933 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/A in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:34,933 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/B in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:34,933 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e40e24f01fb64a44a2079a943bfdda18, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/52ac4a71a17e41fc99e4e1dd9df44659, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3dfc6ca839304210837c03f8fd0c6b13] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=39.7 K 2024-12-03T05:01:34,934 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8a63039d8c4646fc8e9b45cbcd86b394, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/97dd869da7fe4c04b1321e9ac0b9ebb9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8ff4e1143926452f8354c96c48769088] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=35.2 K 2024-12-03T05:01:34,935 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting e40e24f01fb64a44a2079a943bfdda18, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733202090755 2024-12-03T05:01:34,935 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a63039d8c4646fc8e9b45cbcd86b394, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733202090765 2024-12-03T05:01:34,936 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 52ac4a71a17e41fc99e4e1dd9df44659, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733202091049 2024-12-03T05:01:34,936 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97dd869da7fe4c04b1321e9ac0b9ebb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733202091049 2024-12-03T05:01:34,937 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ff4e1143926452f8354c96c48769088, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733202092253 2024-12-03T05:01:34,937 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dfc6ca839304210837c03f8fd0c6b13, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733202092249 2024-12-03T05:01:34,962 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:34,963 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/54c2cbd5bca946a5882eb6b8fce9370c is 50, key is test_row_0/B:col10/1733202092253/Put/seqid=0 2024-12-03T05:01:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:01:34,970 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:34,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-03T05:01:34,974 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:01:34,975 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:34,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:34,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742279_1455 (size=12104) 2024-12-03T05:01:35,001 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/54c2cbd5bca946a5882eb6b8fce9370c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/54c2cbd5bca946a5882eb6b8fce9370c 2024-12-03T05:01:35,007 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#A#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:35,008 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/85255ea3cf0c4ce38189fa4fb17eaf3f is 50, key is test_row_0/A:col10/1733202092253/Put/seqid=0 2024-12-03T05:01:35,008 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/B of 1ef6c76f0cc0543400edbcde21d158fa into 54c2cbd5bca946a5882eb6b8fce9370c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:35,009 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:35,009 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/B, priority=13, startTime=1733202094931; duration=0sec 2024-12-03T05:01:35,009 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:35,009 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:B 2024-12-03T05:01:35,009 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:35,010 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:35,010 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/C is initiating minor compaction (all files) 2024-12-03T05:01:35,010 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/C in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:35,010 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/efa5238fe5104c6dbb4c6b83d23dc774, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/8aefb36cfd754f1890f1b12d9722e8d0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/351f266580f946059b3f5c591a56d302] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=35.2 K 2024-12-03T05:01:35,011 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting efa5238fe5104c6dbb4c6b83d23dc774, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733202090765 2024-12-03T05:01:35,011 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aefb36cfd754f1890f1b12d9722e8d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733202091049 2024-12-03T05:01:35,012 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 351f266580f946059b3f5c591a56d302, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733202092253 2024-12-03T05:01:35,023 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:35,024 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/6b4ae19a8a0043c6aebf7c120a18ed1e is 50, key is test_row_0/C:col10/1733202092253/Put/seqid=0 2024-12-03T05:01:35,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742280_1456 (size=12104) 2024-12-03T05:01:35,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742281_1457 (size=12104) 2024-12-03T05:01:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:01:35,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T05:01:35,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:35,134 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:01:35,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/9b0f0aae1dc44d4381ae3d458da1c839 is 50, key is test_row_0/A:col10/1733202094517/Put/seqid=0 2024-12-03T05:01:35,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742282_1458 (size=12001) 2024-12-03T05:01:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:35,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:35,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202155211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202155217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202155218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:01:35,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33850 deadline: 1733202155290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,299 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_1' in 1ef6c76f0cc0543400edbcde21d158fa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:35,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202155300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,310 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in 1ef6c76f0cc0543400edbcde21d158fa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:35,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202155321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202155340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202155341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,467 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/85255ea3cf0c4ce38189fa4fb17eaf3f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/85255ea3cf0c4ce38189fa4fb17eaf3f 2024-12-03T05:01:35,474 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/A of 1ef6c76f0cc0543400edbcde21d158fa into 85255ea3cf0c4ce38189fa4fb17eaf3f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:35,474 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:35,474 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/A, priority=13, startTime=1733202094931; duration=0sec 2024-12-03T05:01:35,474 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:35,474 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:A 2024-12-03T05:01:35,480 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/6b4ae19a8a0043c6aebf7c120a18ed1e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/6b4ae19a8a0043c6aebf7c120a18ed1e 2024-12-03T05:01:35,485 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/C of 1ef6c76f0cc0543400edbcde21d158fa into 6b4ae19a8a0043c6aebf7c120a18ed1e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:35,485 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:35,485 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/C, priority=13, startTime=1733202094931; duration=0sec 2024-12-03T05:01:35,485 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:35,485 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:C 2024-12-03T05:01:35,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202155531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,556 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/9b0f0aae1dc44d4381ae3d458da1c839 2024-12-03T05:01:35,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202155555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202155561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/6e4dd7f83b6c4c67a5b364cdb729192d is 50, key is test_row_0/B:col10/1733202094517/Put/seqid=0 2024-12-03T05:01:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:01:35,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742283_1459 (size=12001) 2024-12-03T05:01:35,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202155843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202155880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202155880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:36,008 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/6e4dd7f83b6c4c67a5b364cdb729192d 2024-12-03T05:01:36,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/88a303b5f85843dfb2c10966d444743a is 50, key is test_row_0/C:col10/1733202094517/Put/seqid=0 2024-12-03T05:01:36,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742284_1460 (size=12001) 2024-12-03T05:01:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:01:36,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202156362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:36,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202156401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:36,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202156401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:36,426 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/88a303b5f85843dfb2c10966d444743a 2024-12-03T05:01:36,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/9b0f0aae1dc44d4381ae3d458da1c839 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/9b0f0aae1dc44d4381ae3d458da1c839 2024-12-03T05:01:36,442 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/9b0f0aae1dc44d4381ae3d458da1c839, entries=150, sequenceid=75, filesize=11.7 K 2024-12-03T05:01:36,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/6e4dd7f83b6c4c67a5b364cdb729192d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6e4dd7f83b6c4c67a5b364cdb729192d 2024-12-03T05:01:36,446 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6e4dd7f83b6c4c67a5b364cdb729192d, entries=150, sequenceid=75, filesize=11.7 K 2024-12-03T05:01:36,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/88a303b5f85843dfb2c10966d444743a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/88a303b5f85843dfb2c10966d444743a 2024-12-03T05:01:36,451 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/88a303b5f85843dfb2c10966d444743a, entries=150, sequenceid=75, filesize=11.7 K 2024-12-03T05:01:36,453 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1ef6c76f0cc0543400edbcde21d158fa in 1319ms, sequenceid=75, compaction requested=false 2024-12-03T05:01:36,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:36,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:36,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-03T05:01:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-03T05:01:36,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-03T05:01:36,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4800 sec 2024-12-03T05:01:36,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4870 sec 2024-12-03T05:01:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:01:37,122 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:37,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-03T05:01:37,125 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:37,126 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:37,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T05:01:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T05:01:37,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T05:01:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:37,290 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:01:37,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/572187e44f8f47148602814f8b03a70e is 50, key is test_row_0/A:col10/1733202095199/Put/seqid=0 2024-12-03T05:01:37,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742285_1461 (size=12001) 2024-12-03T05:01:37,357 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/572187e44f8f47148602814f8b03a70e 2024-12-03T05:01:37,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/961583f9d8d9473bbbb8c6029ae4078c is 50, key is test_row_0/B:col10/1733202095199/Put/seqid=0 2024-12-03T05:01:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:37,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742286_1462 (size=12001) 2024-12-03T05:01:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T05:01:37,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202157543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:37,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202157552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:37,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202157554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:37,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202157660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:37,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202157670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202157670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T05:01:37,798 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/961583f9d8d9473bbbb8c6029ae4078c 2024-12-03T05:01:37,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/f817747c957e4a0dbc24eb9e27c7ba28 is 50, key is test_row_0/C:col10/1733202095199/Put/seqid=0 2024-12-03T05:01:37,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742287_1463 (size=12001) 2024-12-03T05:01:37,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202157880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:37,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:37,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202157880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:37,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:37,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202157880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:38,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202158190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:38,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202158190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:38,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202158201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:38,211 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/f817747c957e4a0dbc24eb9e27c7ba28 2024-12-03T05:01:38,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/572187e44f8f47148602814f8b03a70e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/572187e44f8f47148602814f8b03a70e 2024-12-03T05:01:38,218 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/572187e44f8f47148602814f8b03a70e, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T05:01:38,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/961583f9d8d9473bbbb8c6029ae4078c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/961583f9d8d9473bbbb8c6029ae4078c 2024-12-03T05:01:38,222 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/961583f9d8d9473bbbb8c6029ae4078c, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T05:01:38,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/f817747c957e4a0dbc24eb9e27c7ba28 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/f817747c957e4a0dbc24eb9e27c7ba28 2024-12-03T05:01:38,226 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/f817747c957e4a0dbc24eb9e27c7ba28, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T05:01:38,226 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1ef6c76f0cc0543400edbcde21d158fa in 936ms, sequenceid=88, compaction requested=true 2024-12-03T05:01:38,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:38,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:38,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-03T05:01:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-03T05:01:38,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-03T05:01:38,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1020 sec 2024-12-03T05:01:38,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.1070 sec 2024-12-03T05:01:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T05:01:38,261 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:38,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-03T05:01:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T05:01:38,263 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:38,263 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:38,263 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T05:01:38,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T05:01:38,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:38,416 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:01:38,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/7daaf89d9af940a7b2be4d3902ff7944 is 50, key is test_row_0/A:col10/1733202097542/Put/seqid=0 2024-12-03T05:01:38,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742288_1464 (size=12001) 2024-12-03T05:01:38,438 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/7daaf89d9af940a7b2be4d3902ff7944 2024-12-03T05:01:38,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/0f5941993c514903b925d34aa3fce91e is 50, key is test_row_0/B:col10/1733202097542/Put/seqid=0 2024-12-03T05:01:38,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742289_1465 (size=12001) 2024-12-03T05:01:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T05:01:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:38,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:38,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202158732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:38,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202158733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:38,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202158734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:38,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202158850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:38,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202158851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:38,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202158851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:38,861 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/0f5941993c514903b925d34aa3fce91e 2024-12-03T05:01:38,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/90d4abec6ef34efba9d16dfae3134aa5 is 50, key is test_row_0/C:col10/1733202097542/Put/seqid=0 2024-12-03T05:01:38,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742290_1466 (size=12001) 2024-12-03T05:01:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T05:01:39,018 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e40e24f01fb64a44a2079a943bfdda18, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/52ac4a71a17e41fc99e4e1dd9df44659, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3dfc6ca839304210837c03f8fd0c6b13] to archive 2024-12-03T05:01:39,019 WARN [regionserver/32beccd76b35:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-12-03T05:01:39,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:01:39,024 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e40e24f01fb64a44a2079a943bfdda18 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e40e24f01fb64a44a2079a943bfdda18 2024-12-03T05:01:39,030 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/52ac4a71a17e41fc99e4e1dd9df44659 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/52ac4a71a17e41fc99e4e1dd9df44659 2024-12-03T05:01:39,031 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3dfc6ca839304210837c03f8fd0c6b13 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3dfc6ca839304210837c03f8fd0c6b13 2024-12-03T05:01:39,032 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8a63039d8c4646fc8e9b45cbcd86b394, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/97dd869da7fe4c04b1321e9ac0b9ebb9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8ff4e1143926452f8354c96c48769088] to archive 2024-12-03T05:01:39,033 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:01:39,035 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8a63039d8c4646fc8e9b45cbcd86b394 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8a63039d8c4646fc8e9b45cbcd86b394 2024-12-03T05:01:39,036 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/97dd869da7fe4c04b1321e9ac0b9ebb9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/97dd869da7fe4c04b1321e9ac0b9ebb9 2024-12-03T05:01:39,039 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8ff4e1143926452f8354c96c48769088 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/8ff4e1143926452f8354c96c48769088 2024-12-03T05:01:39,040 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/efa5238fe5104c6dbb4c6b83d23dc774, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/8aefb36cfd754f1890f1b12d9722e8d0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/351f266580f946059b3f5c591a56d302] to archive 2024-12-03T05:01:39,049 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:01:39,052 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/efa5238fe5104c6dbb4c6b83d23dc774 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/efa5238fe5104c6dbb4c6b83d23dc774 2024-12-03T05:01:39,053 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/8aefb36cfd754f1890f1b12d9722e8d0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/8aefb36cfd754f1890f1b12d9722e8d0 2024-12-03T05:01:39,055 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/351f266580f946059b3f5c591a56d302 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/351f266580f946059b3f5c591a56d302 2024-12-03T05:01:39,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202159060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202159060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202159070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,286 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/90d4abec6ef34efba9d16dfae3134aa5 2024-12-03T05:01:39,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/7daaf89d9af940a7b2be4d3902ff7944 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/7daaf89d9af940a7b2be4d3902ff7944 2024-12-03T05:01:39,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33850 deadline: 1733202159322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,335 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_1' in 1ef6c76f0cc0543400edbcde21d158fa of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8284 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,338 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/7daaf89d9af940a7b2be4d3902ff7944, entries=150, sequenceid=113, filesize=11.7 K 2024-12-03T05:01:39,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/0f5941993c514903b925d34aa3fce91e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/0f5941993c514903b925d34aa3fce91e 2024-12-03T05:01:39,343 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/0f5941993c514903b925d34aa3fce91e, entries=150, sequenceid=113, filesize=11.7 K 2024-12-03T05:01:39,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/90d4abec6ef34efba9d16dfae3134aa5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/90d4abec6ef34efba9d16dfae3134aa5 2024-12-03T05:01:39,353 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/90d4abec6ef34efba9d16dfae3134aa5, entries=150, sequenceid=113, filesize=11.7 K 2024-12-03T05:01:39,359 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1ef6c76f0cc0543400edbcde21d158fa in 943ms, sequenceid=113, compaction requested=true 2024-12-03T05:01:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-03T05:01:39,361 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:39,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-03T05:01:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-03T05:01:39,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/75cc714d2ff04b7ab05a61ab6fa00870 is 50, key is test_row_0/A:col10/1733202098718/Put/seqid=0 2024-12-03T05:01:39,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-03T05:01:39,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1060 sec 2024-12-03T05:01:39,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.1250 sec 2024-12-03T05:01:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T05:01:39,401 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:39,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:39,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742291_1467 (size=14341) 2024-12-03T05:01:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-03T05:01:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:01:39,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:39,415 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:39,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202159492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202159493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202159499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:01:39,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202159516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:01:39,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:39,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:39,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:39,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:39,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:39,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202159625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202159625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202159633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202159643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:01:39,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:39,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:39,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:39,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:39,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:01:39,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/75cc714d2ff04b7ab05a61ab6fa00870 2024-12-03T05:01:39,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/6221978d27064ff09f31feabcade032c is 50, key is test_row_0/B:col10/1733202098718/Put/seqid=0 2024-12-03T05:01:39,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202159848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202159848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202159850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:01:39,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:39,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:39,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:39,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:39,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:39,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202159874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:39,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742292_1468 (size=12001) 2024-12-03T05:01:39,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/6221978d27064ff09f31feabcade032c 2024-12-03T05:01:39,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/eb9496890dbf418fb7176ee37bf10045 is 50, key is test_row_0/C:col10/1733202098718/Put/seqid=0 2024-12-03T05:01:39,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742293_1469 (size=12001) 2024-12-03T05:01:40,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:01:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:40,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:01:40,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202160170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:40,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202160170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:40,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202160180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:40,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:01:40,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:40,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:40,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:40,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:40,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:40,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202160190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:40,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/eb9496890dbf418fb7176ee37bf10045 2024-12-03T05:01:40,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/75cc714d2ff04b7ab05a61ab6fa00870 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/75cc714d2ff04b7ab05a61ab6fa00870 2024-12-03T05:01:40,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/75cc714d2ff04b7ab05a61ab6fa00870, entries=200, sequenceid=125, filesize=14.0 K 2024-12-03T05:01:40,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/6221978d27064ff09f31feabcade032c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6221978d27064ff09f31feabcade032c 2024-12-03T05:01:40,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6221978d27064ff09f31feabcade032c, entries=150, sequenceid=125, filesize=11.7 K 2024-12-03T05:01:40,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/eb9496890dbf418fb7176ee37bf10045 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/eb9496890dbf418fb7176ee37bf10045 2024-12-03T05:01:40,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/eb9496890dbf418fb7176ee37bf10045, entries=150, sequenceid=125, filesize=11.7 K 2024-12-03T05:01:40,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1ef6c76f0cc0543400edbcde21d158fa in 970ms, sequenceid=125, compaction requested=true 2024-12-03T05:01:40,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:40,331 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:01:40,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:40,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:40,332 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:01:40,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:40,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:40,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:40,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:40,333 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62448 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:01:40,333 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/A is initiating minor compaction (all files) 2024-12-03T05:01:40,333 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/A in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:40,333 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60108 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:01:40,333 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/85255ea3cf0c4ce38189fa4fb17eaf3f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/9b0f0aae1dc44d4381ae3d458da1c839, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/572187e44f8f47148602814f8b03a70e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/7daaf89d9af940a7b2be4d3902ff7944, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/75cc714d2ff04b7ab05a61ab6fa00870] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=61.0 K 2024-12-03T05:01:40,333 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/B is initiating minor compaction (all files) 2024-12-03T05:01:40,333 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/B in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:40,333 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/54c2cbd5bca946a5882eb6b8fce9370c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6e4dd7f83b6c4c67a5b364cdb729192d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/961583f9d8d9473bbbb8c6029ae4078c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/0f5941993c514903b925d34aa3fce91e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6221978d27064ff09f31feabcade032c] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=58.7 K 2024-12-03T05:01:40,334 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 54c2cbd5bca946a5882eb6b8fce9370c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733202092253 2024-12-03T05:01:40,334 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85255ea3cf0c4ce38189fa4fb17eaf3f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733202092253 2024-12-03T05:01:40,334 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e4dd7f83b6c4c67a5b364cdb729192d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733202094517 2024-12-03T05:01:40,334 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b0f0aae1dc44d4381ae3d458da1c839, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733202094517 2024-12-03T05:01:40,335 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 961583f9d8d9473bbbb8c6029ae4078c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733202095199 2024-12-03T05:01:40,335 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 572187e44f8f47148602814f8b03a70e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733202095199 2024-12-03T05:01:40,335 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f5941993c514903b925d34aa3fce91e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733202097520 2024-12-03T05:01:40,335 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7daaf89d9af940a7b2be4d3902ff7944, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733202097520 2024-12-03T05:01:40,335 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75cc714d2ff04b7ab05a61ab6fa00870, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202098718 2024-12-03T05:01:40,335 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6221978d27064ff09f31feabcade032c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202098718 2024-12-03T05:01:40,347 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#B#compaction#391 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:40,347 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#A#compaction#390 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:40,347 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/363bdcad095a431090ddb21e1f317e8f is 50, key is test_row_0/B:col10/1733202098718/Put/seqid=0 2024-12-03T05:01:40,347 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/10f89e7353e84a23a1a5d9051329173d is 50, key is test_row_0/A:col10/1733202098718/Put/seqid=0 2024-12-03T05:01:40,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:01:40,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:40,354 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-03T05:01:40,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742294_1470 (size=12173) 2024-12-03T05:01:40,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/3c1b94a551c441029debceb7ab5ecd2b is 50, key is test_row_0/A:col10/1733202099490/Put/seqid=0 2024-12-03T05:01:40,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742295_1471 (size=12173) 2024-12-03T05:01:40,375 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/10f89e7353e84a23a1a5d9051329173d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/10f89e7353e84a23a1a5d9051329173d 2024-12-03T05:01:40,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742296_1472 (size=12151) 2024-12-03T05:01:40,376 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/3c1b94a551c441029debceb7ab5ecd2b 2024-12-03T05:01:40,379 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/A of 1ef6c76f0cc0543400edbcde21d158fa into 10f89e7353e84a23a1a5d9051329173d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:40,379 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:40,379 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/A, priority=11, startTime=1733202100331; duration=0sec 2024-12-03T05:01:40,379 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:40,379 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:A 2024-12-03T05:01:40,379 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:01:40,381 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60108 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:01:40,381 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/C is initiating minor compaction (all files) 2024-12-03T05:01:40,381 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/C in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:40,381 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/6b4ae19a8a0043c6aebf7c120a18ed1e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/88a303b5f85843dfb2c10966d444743a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/f817747c957e4a0dbc24eb9e27c7ba28, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/90d4abec6ef34efba9d16dfae3134aa5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/eb9496890dbf418fb7176ee37bf10045] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=58.7 K 2024-12-03T05:01:40,382 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b4ae19a8a0043c6aebf7c120a18ed1e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733202092253 2024-12-03T05:01:40,382 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88a303b5f85843dfb2c10966d444743a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733202094517 2024-12-03T05:01:40,382 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f817747c957e4a0dbc24eb9e27c7ba28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733202095199 2024-12-03T05:01:40,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/cb9c55fc64da4800a09ddb44b8c9d9fc is 50, key is test_row_0/B:col10/1733202099490/Put/seqid=0 2024-12-03T05:01:40,382 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90d4abec6ef34efba9d16dfae3134aa5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733202097520 2024-12-03T05:01:40,383 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb9496890dbf418fb7176ee37bf10045, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202098718 2024-12-03T05:01:40,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742297_1473 (size=12151) 2024-12-03T05:01:40,388 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/cb9c55fc64da4800a09ddb44b8c9d9fc 2024-12-03T05:01:40,392 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#C#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:40,393 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/7ab2d7928a8b4cfb90231d864f24139e is 50, key is test_row_0/C:col10/1733202098718/Put/seqid=0 2024-12-03T05:01:40,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/53d90e61f78c4f239721c048728645e0 is 50, key is test_row_0/C:col10/1733202099490/Put/seqid=0 2024-12-03T05:01:40,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742298_1474 (size=12173) 2024-12-03T05:01:40,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742299_1475 (size=12151) 2024-12-03T05:01:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:01:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:40,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:40,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202160709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202160711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:40,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202160712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:40,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202160722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:40,767 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/363bdcad095a431090ddb21e1f317e8f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/363bdcad095a431090ddb21e1f317e8f 2024-12-03T05:01:40,771 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/B of 1ef6c76f0cc0543400edbcde21d158fa into 363bdcad095a431090ddb21e1f317e8f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:40,771 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:40,771 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/B, priority=11, startTime=1733202100331; duration=0sec 2024-12-03T05:01:40,771 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:40,771 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:B 2024-12-03T05:01:40,807 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/7ab2d7928a8b4cfb90231d864f24139e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/7ab2d7928a8b4cfb90231d864f24139e 2024-12-03T05:01:40,812 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/53d90e61f78c4f239721c048728645e0 2024-12-03T05:01:40,812 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/C of 1ef6c76f0cc0543400edbcde21d158fa into 7ab2d7928a8b4cfb90231d864f24139e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:40,812 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:40,813 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/C, priority=11, startTime=1733202100332; duration=0sec 2024-12-03T05:01:40,813 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:40,813 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:C 2024-12-03T05:01:40,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/3c1b94a551c441029debceb7ab5ecd2b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3c1b94a551c441029debceb7ab5ecd2b 2024-12-03T05:01:40,819 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3c1b94a551c441029debceb7ab5ecd2b, entries=150, sequenceid=149, filesize=11.9 K 2024-12-03T05:01:40,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/cb9c55fc64da4800a09ddb44b8c9d9fc as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/cb9c55fc64da4800a09ddb44b8c9d9fc 2024-12-03T05:01:40,824 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/cb9c55fc64da4800a09ddb44b8c9d9fc, entries=150, sequenceid=149, filesize=11.9 K 2024-12-03T05:01:40,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/53d90e61f78c4f239721c048728645e0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/53d90e61f78c4f239721c048728645e0 2024-12-03T05:01:40,827 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/53d90e61f78c4f239721c048728645e0, entries=150, sequenceid=149, filesize=11.9 K 2024-12-03T05:01:40,828 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1ef6c76f0cc0543400edbcde21d158fa in 474ms, sequenceid=149, compaction requested=false 2024-12-03T05:01:40,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:40,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:40,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-03T05:01:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-03T05:01:40,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-03T05:01:40,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-12-03T05:01:40,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4320 sec 2024-12-03T05:01:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:40,835 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:40,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-03T05:01:40,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/72cc77402e9b42389d5db0a40ddbd91d is 50, key is test_row_0/A:col10/1733202100834/Put/seqid=0 2024-12-03T05:01:40,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742300_1476 (size=12151) 2024-12-03T05:01:40,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/72cc77402e9b42389d5db0a40ddbd91d 2024-12-03T05:01:40,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/1bbae731948e4df58302cfa0fb9f5ab9 is 50, key is test_row_0/B:col10/1733202100834/Put/seqid=0 2024-12-03T05:01:40,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742301_1477 (size=12151) 2024-12-03T05:01:40,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202160917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202160918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:40,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202160928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202161040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202161040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202161041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202161251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202161252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202161252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/1bbae731948e4df58302cfa0fb9f5ab9 2024-12-03T05:01:41,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/e0342cbf799a4917a881ec436d4f20d3 is 50, key is test_row_0/C:col10/1733202100834/Put/seqid=0 2024-12-03T05:01:41,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742302_1478 (size=12151) 2024-12-03T05:01:41,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/e0342cbf799a4917a881ec436d4f20d3 2024-12-03T05:01:41,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/72cc77402e9b42389d5db0a40ddbd91d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/72cc77402e9b42389d5db0a40ddbd91d 2024-12-03T05:01:41,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/72cc77402e9b42389d5db0a40ddbd91d, entries=150, sequenceid=165, filesize=11.9 K 2024-12-03T05:01:41,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/1bbae731948e4df58302cfa0fb9f5ab9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/1bbae731948e4df58302cfa0fb9f5ab9 2024-12-03T05:01:41,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/1bbae731948e4df58302cfa0fb9f5ab9, entries=150, sequenceid=165, filesize=11.9 K 2024-12-03T05:01:41,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/e0342cbf799a4917a881ec436d4f20d3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/e0342cbf799a4917a881ec436d4f20d3 2024-12-03T05:01:41,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/e0342cbf799a4917a881ec436d4f20d3, entries=150, sequenceid=165, filesize=11.9 K 2024-12-03T05:01:41,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1ef6c76f0cc0543400edbcde21d158fa in 584ms, sequenceid=165, compaction requested=true 2024-12-03T05:01:41,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:41,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:41,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:41,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:41,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:41,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:41,419 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:41,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:41,419 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:41,426 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36475 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:41,426 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/B is initiating minor compaction (all files) 2024-12-03T05:01:41,426 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/B in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:41,426 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/363bdcad095a431090ddb21e1f317e8f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/cb9c55fc64da4800a09ddb44b8c9d9fc, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/1bbae731948e4df58302cfa0fb9f5ab9] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=35.6 K 2024-12-03T05:01:41,426 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36475 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:41,426 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/A is initiating minor compaction (all files) 2024-12-03T05:01:41,426 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/A in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:41,426 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/10f89e7353e84a23a1a5d9051329173d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3c1b94a551c441029debceb7ab5ecd2b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/72cc77402e9b42389d5db0a40ddbd91d] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=35.6 K 2024-12-03T05:01:41,427 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 363bdcad095a431090ddb21e1f317e8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202098718 2024-12-03T05:01:41,427 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10f89e7353e84a23a1a5d9051329173d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202098718 2024-12-03T05:01:41,427 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting cb9c55fc64da4800a09ddb44b8c9d9fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733202099489 2024-12-03T05:01:41,428 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c1b94a551c441029debceb7ab5ecd2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733202099489 2024-12-03T05:01:41,428 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bbae731948e4df58302cfa0fb9f5ab9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733202100711 2024-12-03T05:01:41,428 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72cc77402e9b42389d5db0a40ddbd91d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733202100711 2024-12-03T05:01:41,438 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#A#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:41,439 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/a6dc5f6ab4424d27b856b486a037d32b is 50, key is test_row_0/A:col10/1733202100834/Put/seqid=0 2024-12-03T05:01:41,439 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:41,439 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/7c647e87845449e3a22e95d1fed3d359 is 50, key is test_row_0/B:col10/1733202100834/Put/seqid=0 2024-12-03T05:01:41,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742303_1479 (size=12425) 2024-12-03T05:01:41,461 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/7c647e87845449e3a22e95d1fed3d359 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7c647e87845449e3a22e95d1fed3d359 2024-12-03T05:01:41,467 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/B of 1ef6c76f0cc0543400edbcde21d158fa into 7c647e87845449e3a22e95d1fed3d359(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:41,467 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:41,467 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/B, priority=13, startTime=1733202101419; duration=0sec 2024-12-03T05:01:41,467 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:41,467 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:B 2024-12-03T05:01:41,467 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:41,469 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36475 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:41,469 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/C is initiating minor compaction (all files) 2024-12-03T05:01:41,469 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/C in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:41,469 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/7ab2d7928a8b4cfb90231d864f24139e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/53d90e61f78c4f239721c048728645e0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/e0342cbf799a4917a881ec436d4f20d3] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=35.6 K 2024-12-03T05:01:41,470 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ab2d7928a8b4cfb90231d864f24139e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202098718 2024-12-03T05:01:41,473 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d90e61f78c4f239721c048728645e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733202099489 2024-12-03T05:01:41,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742304_1480 (size=12425) 2024-12-03T05:01:41,474 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting e0342cbf799a4917a881ec436d4f20d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733202100711 2024-12-03T05:01:41,509 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:41,509 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/3e0250c3812d42c59122b09bb262125b is 50, key is test_row_0/C:col10/1733202100834/Put/seqid=0 2024-12-03T05:01:41,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742305_1481 (size=12425) 2024-12-03T05:01:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:01:41,561 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:41,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-03T05:01:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T05:01:41,563 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:41,564 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:41,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:41,564 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:41,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-03T05:01:41,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/aa6ef64ab8254832b79f061992ab151f is 50, key is test_row_0/A:col10/1733202101563/Put/seqid=0 2024-12-03T05:01:41,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742306_1482 (size=14541) 2024-12-03T05:01:41,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202161585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202161590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202161591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T05:01:41,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202161701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T05:01:41,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:41,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:41,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:41,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:41,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:41,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202161710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202161710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202161730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,740 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_2' in 1ef6c76f0cc0543400edbcde21d158fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T05:01:41,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:41,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:41,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:41,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:41,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:41,877 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/a6dc5f6ab4424d27b856b486a037d32b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a6dc5f6ab4424d27b856b486a037d32b 2024-12-03T05:01:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T05:01:41,881 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/A of 1ef6c76f0cc0543400edbcde21d158fa into a6dc5f6ab4424d27b856b486a037d32b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:41,881 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:41,881 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/A, priority=13, startTime=1733202101419; duration=0sec 2024-12-03T05:01:41,881 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:41,881 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:A 2024-12-03T05:01:41,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202161910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,926 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/3e0250c3812d42c59122b09bb262125b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/3e0250c3812d42c59122b09bb262125b 2024-12-03T05:01:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202161920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:41,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202161921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:41,930 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/C of 1ef6c76f0cc0543400edbcde21d158fa into 3e0250c3812d42c59122b09bb262125b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:41,930 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:41,930 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/C, priority=13, startTime=1733202101419; duration=0sec 2024-12-03T05:01:41,930 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:41,930 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:C 2024-12-03T05:01:41,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/aa6ef64ab8254832b79f061992ab151f 2024-12-03T05:01:41,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/6b23f30432234f1d987f56f6ab2f0c0f is 50, key is test_row_0/B:col10/1733202101563/Put/seqid=0 2024-12-03T05:01:41,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742307_1483 (size=12151) 2024-12-03T05:01:42,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T05:01:42,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:42,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T05:01:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T05:01:42,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202162222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:42,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202162234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202162234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:42,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T05:01:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/6b23f30432234f1d987f56f6ab2f0c0f 2024-12-03T05:01:42,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/bbf33966815f49d3896ae38c5dbc7d51 is 50, key is test_row_0/C:col10/1733202101563/Put/seqid=0 2024-12-03T05:01:42,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742308_1484 (size=12151) 2024-12-03T05:01:42,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T05:01:42,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:42,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T05:01:42,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:42,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T05:01:42,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202162741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:42,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:42,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202162751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:42,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202162751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:42,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T05:01:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:42,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/bbf33966815f49d3896ae38c5dbc7d51 2024-12-03T05:01:42,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/aa6ef64ab8254832b79f061992ab151f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/aa6ef64ab8254832b79f061992ab151f 2024-12-03T05:01:42,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/aa6ef64ab8254832b79f061992ab151f, entries=200, sequenceid=190, filesize=14.2 K 2024-12-03T05:01:42,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/6b23f30432234f1d987f56f6ab2f0c0f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6b23f30432234f1d987f56f6ab2f0c0f 2024-12-03T05:01:42,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6b23f30432234f1d987f56f6ab2f0c0f, entries=150, sequenceid=190, filesize=11.9 K 2024-12-03T05:01:42,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/bbf33966815f49d3896ae38c5dbc7d51 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/bbf33966815f49d3896ae38c5dbc7d51 2024-12-03T05:01:42,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/bbf33966815f49d3896ae38c5dbc7d51, entries=150, sequenceid=190, filesize=11.9 K 2024-12-03T05:01:42,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1ef6c76f0cc0543400edbcde21d158fa in 1357ms, sequenceid=190, compaction requested=false 2024-12-03T05:01:42,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:42,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T05:01:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:42,957 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-03T05:01:42,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/b0bb828726834f2fb5a01be93aba74c4 is 50, key is test_row_0/A:col10/1733202101582/Put/seqid=0 2024-12-03T05:01:42,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742309_1485 (size=12151) 2024-12-03T05:01:43,369 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/b0bb828726834f2fb5a01be93aba74c4 2024-12-03T05:01:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/dfd753c3571e4ff1b3d1ac6f201e4467 is 50, key is test_row_0/B:col10/1733202101582/Put/seqid=0 2024-12-03T05:01:43,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742310_1486 (size=12151) 2024-12-03T05:01:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T05:01:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:43,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:43,780 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/dfd753c3571e4ff1b3d1ac6f201e4467 2024-12-03T05:01:43,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/77934522fbd84291a3ae8cd97ceadb8c is 50, key is test_row_0/C:col10/1733202101582/Put/seqid=0 2024-12-03T05:01:43,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742311_1487 (size=12151) 2024-12-03T05:01:43,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202163822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:43,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202163823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:43,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202163824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202163827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202163940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202163940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202163940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:43,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202163940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202164151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202164151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202164151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202164151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,190 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/77934522fbd84291a3ae8cd97ceadb8c 2024-12-03T05:01:44,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/b0bb828726834f2fb5a01be93aba74c4 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b0bb828726834f2fb5a01be93aba74c4 2024-12-03T05:01:44,197 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b0bb828726834f2fb5a01be93aba74c4, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T05:01:44,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/dfd753c3571e4ff1b3d1ac6f201e4467 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dfd753c3571e4ff1b3d1ac6f201e4467 2024-12-03T05:01:44,203 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dfd753c3571e4ff1b3d1ac6f201e4467, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T05:01:44,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/77934522fbd84291a3ae8cd97ceadb8c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/77934522fbd84291a3ae8cd97ceadb8c 2024-12-03T05:01:44,207 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/77934522fbd84291a3ae8cd97ceadb8c, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T05:01:44,208 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1ef6c76f0cc0543400edbcde21d158fa in 1250ms, sequenceid=204, compaction requested=true 2024-12-03T05:01:44,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:44,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:44,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-03T05:01:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-03T05:01:44,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-03T05:01:44,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6450 sec 2024-12-03T05:01:44,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.6510 sec 2024-12-03T05:01:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:44,476 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:44,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:01:44,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/c82330d7a3a1493185348f941c1006c6 is 50, key is test_row_0/A:col10/1733202104474/Put/seqid=0 2024-12-03T05:01:44,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742312_1488 (size=14541) 2024-12-03T05:01:44,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/c82330d7a3a1493185348f941c1006c6 2024-12-03T05:01:44,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202164494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202164494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202164495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202164500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/a8dcf3b806db4f9e830e9c4b4d77b320 is 50, key is test_row_0/B:col10/1733202104474/Put/seqid=0 2024-12-03T05:01:44,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742313_1489 (size=12151) 2024-12-03T05:01:44,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202164620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202164620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202164620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202164620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202164830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202164830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202164841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202164841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:44,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/a8dcf3b806db4f9e830e9c4b4d77b320 2024-12-03T05:01:44,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/54ef48af6e9c4960bd07936a374826df is 50, key is test_row_0/C:col10/1733202104474/Put/seqid=0 2024-12-03T05:01:44,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742314_1490 (size=12151) 2024-12-03T05:01:44,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/54ef48af6e9c4960bd07936a374826df 2024-12-03T05:01:44,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/c82330d7a3a1493185348f941c1006c6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/c82330d7a3a1493185348f941c1006c6 2024-12-03T05:01:44,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/c82330d7a3a1493185348f941c1006c6, entries=200, sequenceid=229, filesize=14.2 K 2024-12-03T05:01:44,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/a8dcf3b806db4f9e830e9c4b4d77b320 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a8dcf3b806db4f9e830e9c4b4d77b320 2024-12-03T05:01:44,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a8dcf3b806db4f9e830e9c4b4d77b320, entries=150, sequenceid=229, filesize=11.9 K 2024-12-03T05:01:44,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/54ef48af6e9c4960bd07936a374826df as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/54ef48af6e9c4960bd07936a374826df 2024-12-03T05:01:44,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/54ef48af6e9c4960bd07936a374826df, entries=150, sequenceid=229, filesize=11.9 K 2024-12-03T05:01:44,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1ef6c76f0cc0543400edbcde21d158fa in 480ms, sequenceid=229, compaction requested=true 2024-12-03T05:01:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:44,956 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:01:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:44,956 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:01:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:44,958 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:01:44,958 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53658 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:01:44,958 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/B is initiating minor compaction (all files) 2024-12-03T05:01:44,958 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/A is initiating minor compaction (all files) 2024-12-03T05:01:44,958 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/B in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:44,958 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/A in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:44,958 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a6dc5f6ab4424d27b856b486a037d32b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/aa6ef64ab8254832b79f061992ab151f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b0bb828726834f2fb5a01be93aba74c4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/c82330d7a3a1493185348f941c1006c6] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=52.4 K 2024-12-03T05:01:44,958 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7c647e87845449e3a22e95d1fed3d359, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6b23f30432234f1d987f56f6ab2f0c0f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dfd753c3571e4ff1b3d1ac6f201e4467, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a8dcf3b806db4f9e830e9c4b4d77b320] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=47.7 K 2024-12-03T05:01:44,959 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6dc5f6ab4424d27b856b486a037d32b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733202100711 2024-12-03T05:01:44,959 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c647e87845449e3a22e95d1fed3d359, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733202100711 2024-12-03T05:01:44,959 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa6ef64ab8254832b79f061992ab151f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733202100910 2024-12-03T05:01:44,959 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b23f30432234f1d987f56f6ab2f0c0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733202100910 2024-12-03T05:01:44,959 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting dfd753c3571e4ff1b3d1ac6f201e4467, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733202101582 2024-12-03T05:01:44,959 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0bb828726834f2fb5a01be93aba74c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733202101582 2024-12-03T05:01:44,960 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c82330d7a3a1493185348f941c1006c6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733202103821 2024-12-03T05:01:44,960 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a8dcf3b806db4f9e830e9c4b4d77b320, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733202103821 2024-12-03T05:01:44,968 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#A#compaction#411 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:44,968 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#B#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:44,968 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/16603526217e4d77a8814bd818f1575e is 50, key is test_row_0/A:col10/1733202104474/Put/seqid=0 2024-12-03T05:01:44,969 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/54f50240d9fe42dab09763dab39b2a70 is 50, key is test_row_0/B:col10/1733202104474/Put/seqid=0 2024-12-03T05:01:44,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742316_1492 (size=12561) 2024-12-03T05:01:44,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742315_1491 (size=12561) 2024-12-03T05:01:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:45,161 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:45,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-03T05:01:45,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/e5fb9a9dbb4d40c5ba450006f083666e is 50, key is test_row_0/A:col10/1733202105158/Put/seqid=0 2024-12-03T05:01:45,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742317_1493 (size=14541) 2024-12-03T05:01:45,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/e5fb9a9dbb4d40c5ba450006f083666e 2024-12-03T05:01:45,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/7818fdafb07a407fa4c209161f59e543 is 50, key is test_row_0/B:col10/1733202105158/Put/seqid=0 2024-12-03T05:01:45,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742318_1494 (size=12151) 2024-12-03T05:01:45,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/7818fdafb07a407fa4c209161f59e543 2024-12-03T05:01:45,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/44d0d80f4a4a49bfb7f4c0e422e9cb56 is 50, key is test_row_0/C:col10/1733202105158/Put/seqid=0 2024-12-03T05:01:45,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742319_1495 (size=12151) 2024-12-03T05:01:45,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/44d0d80f4a4a49bfb7f4c0e422e9cb56 2024-12-03T05:01:45,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/e5fb9a9dbb4d40c5ba450006f083666e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e5fb9a9dbb4d40c5ba450006f083666e 2024-12-03T05:01:45,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e5fb9a9dbb4d40c5ba450006f083666e, entries=200, sequenceid=241, filesize=14.2 K 2024-12-03T05:01:45,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/7818fdafb07a407fa4c209161f59e543 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7818fdafb07a407fa4c209161f59e543 2024-12-03T05:01:45,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7818fdafb07a407fa4c209161f59e543, entries=150, sequenceid=241, filesize=11.9 K 2024-12-03T05:01:45,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/44d0d80f4a4a49bfb7f4c0e422e9cb56 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/44d0d80f4a4a49bfb7f4c0e422e9cb56 2024-12-03T05:01:45,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202165251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202165258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202165260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202165269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/44d0d80f4a4a49bfb7f4c0e422e9cb56, entries=150, sequenceid=241, filesize=11.9 K 2024-12-03T05:01:45,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1ef6c76f0cc0543400edbcde21d158fa in 116ms, sequenceid=241, compaction requested=true 2024-12-03T05:01:45,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:45,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:A, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:45,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:01:45,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:B, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:45,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T05:01:45,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:45,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T05:01:45,382 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/16603526217e4d77a8814bd818f1575e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/16603526217e4d77a8814bd818f1575e 2024-12-03T05:01:45,383 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/54f50240d9fe42dab09763dab39b2a70 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/54f50240d9fe42dab09763dab39b2a70 2024-12-03T05:01:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:45,385 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:45,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:01:45,391 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/B of 1ef6c76f0cc0543400edbcde21d158fa into 54f50240d9fe42dab09763dab39b2a70(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:45,391 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:45,391 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/B, priority=12, startTime=1733202104956; duration=0sec 2024-12-03T05:01:45,391 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T05:01:45,391 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:B 2024-12-03T05:01:45,392 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-12-03T05:01:45,394 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:01:45,394 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:01:45,394 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. because compaction request was cancelled 2024-12-03T05:01:45,394 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:A 2024-12-03T05:01:45,394 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:01:45,395 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/A of 1ef6c76f0cc0543400edbcde21d158fa into 16603526217e4d77a8814bd818f1575e(size=12.3 K), total size for store is 26.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:45,395 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:45,395 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/A, priority=12, startTime=1733202104956; duration=0sec 2024-12-03T05:01:45,395 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:01:45,395 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:A 2024-12-03T05:01:45,395 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:B 2024-12-03T05:01:45,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/3705f483fb024129ab30702948a20af1 is 50, key is test_row_0/A:col10/1733202105382/Put/seqid=0 2024-12-03T05:01:45,399 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61029 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:01:45,400 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/C is initiating minor compaction (all files) 2024-12-03T05:01:45,400 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/C in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:45,400 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/3e0250c3812d42c59122b09bb262125b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/bbf33966815f49d3896ae38c5dbc7d51, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/77934522fbd84291a3ae8cd97ceadb8c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/54ef48af6e9c4960bd07936a374826df, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/44d0d80f4a4a49bfb7f4c0e422e9cb56] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=59.6 K 2024-12-03T05:01:45,400 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-12-03T05:01:45,400 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:01:45,400 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:01:45,400 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. because compaction request was cancelled 2024-12-03T05:01:45,400 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:C 2024-12-03T05:01:45,400 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:01:45,401 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e0250c3812d42c59122b09bb262125b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733202100711 2024-12-03T05:01:45,405 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting bbf33966815f49d3896ae38c5dbc7d51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733202100910 2024-12-03T05:01:45,406 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:01:45,406 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:01:45,406 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. because compaction request was cancelled 2024-12-03T05:01:45,406 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:B 2024-12-03T05:01:45,406 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 77934522fbd84291a3ae8cd97ceadb8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733202101582 2024-12-03T05:01:45,406 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ef48af6e9c4960bd07936a374826df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733202103821 2024-12-03T05:01:45,406 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 44d0d80f4a4a49bfb7f4c0e422e9cb56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733202104489 2024-12-03T05:01:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742320_1496 (size=14691) 2024-12-03T05:01:45,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/3705f483fb024129ab30702948a20af1 2024-12-03T05:01:45,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/46acd714777842a88df76d2fafbd8470 is 50, key is test_row_0/B:col10/1733202105382/Put/seqid=0 2024-12-03T05:01:45,424 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#C#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:45,425 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/4fdcb90235df43b8b00da6a31428ae35 is 50, key is test_row_0/C:col10/1733202105158/Put/seqid=0 2024-12-03T05:01:45,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202165416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202165417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202165420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742321_1497 (size=12301) 2024-12-03T05:01:45,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202165422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/46acd714777842a88df76d2fafbd8470 2024-12-03T05:01:45,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/c833f881005c4e00aa17e1a3e2763de0 is 50, key is test_row_0/C:col10/1733202105382/Put/seqid=0 2024-12-03T05:01:45,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742322_1498 (size=12595) 2024-12-03T05:01:45,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742323_1499 (size=12301) 2024-12-03T05:01:45,473 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/4fdcb90235df43b8b00da6a31428ae35 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/4fdcb90235df43b8b00da6a31428ae35 2024-12-03T05:01:45,483 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/C of 1ef6c76f0cc0543400edbcde21d158fa into 4fdcb90235df43b8b00da6a31428ae35(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:45,484 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:45,484 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/C, priority=11, startTime=1733202105277; duration=0sec 2024-12-03T05:01:45,484 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:45,484 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:C 2024-12-03T05:01:45,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202165541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202165541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202165541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202165541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T05:01:45,730 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:45,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-03T05:01:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:01:45,733 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:45,733 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:45,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:45,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202165750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202165750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202165760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202165760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:01:45,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/c833f881005c4e00aa17e1a3e2763de0 2024-12-03T05:01:45,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/3705f483fb024129ab30702948a20af1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3705f483fb024129ab30702948a20af1 2024-12-03T05:01:45,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3705f483fb024129ab30702948a20af1, entries=200, sequenceid=266, filesize=14.3 K 2024-12-03T05:01:45,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/46acd714777842a88df76d2fafbd8470 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/46acd714777842a88df76d2fafbd8470 2024-12-03T05:01:45,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/46acd714777842a88df76d2fafbd8470, entries=150, sequenceid=266, filesize=12.0 K 2024-12-03T05:01:45,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/c833f881005c4e00aa17e1a3e2763de0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/c833f881005c4e00aa17e1a3e2763de0 2024-12-03T05:01:45,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:01:45,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:45,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:45,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:45,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:45,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:45,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/c833f881005c4e00aa17e1a3e2763de0, entries=150, sequenceid=266, filesize=12.0 K 2024-12-03T05:01:45,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1ef6c76f0cc0543400edbcde21d158fa in 506ms, sequenceid=266, compaction requested=true 2024-12-03T05:01:45,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:45,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:45,891 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:45,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:45,891 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:45,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:45,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:45,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:45,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:45,892 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41793 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:45,892 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/A is initiating minor compaction (all files) 2024-12-03T05:01:45,892 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/A in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:45,892 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:45,892 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/16603526217e4d77a8814bd818f1575e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e5fb9a9dbb4d40c5ba450006f083666e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3705f483fb024129ab30702948a20af1] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=40.8 K 2024-12-03T05:01:45,892 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/B is initiating minor compaction (all files) 2024-12-03T05:01:45,892 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/B in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:45,892 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/54f50240d9fe42dab09763dab39b2a70, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7818fdafb07a407fa4c209161f59e543, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/46acd714777842a88df76d2fafbd8470] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=36.1 K 2024-12-03T05:01:45,892 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16603526217e4d77a8814bd818f1575e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733202103821 2024-12-03T05:01:45,893 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f50240d9fe42dab09763dab39b2a70, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733202103821 2024-12-03T05:01:45,893 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5fb9a9dbb4d40c5ba450006f083666e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733202104488 2024-12-03T05:01:45,893 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7818fdafb07a407fa4c209161f59e543, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733202104489 2024-12-03T05:01:45,893 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3705f483fb024129ab30702948a20af1, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733202105229 2024-12-03T05:01:45,893 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 46acd714777842a88df76d2fafbd8470, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733202105258 2024-12-03T05:01:45,903 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:45,903 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#A#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:45,904 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/b364d9f7895442a087a767700b72b8d4 is 50, key is test_row_0/A:col10/1733202105382/Put/seqid=0 2024-12-03T05:01:45,904 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/dd4ddd95ec934e69993be0392357646c is 50, key is test_row_0/B:col10/1733202105382/Put/seqid=0 2024-12-03T05:01:45,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742324_1500 (size=12813) 2024-12-03T05:01:45,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742325_1501 (size=12813) 2024-12-03T05:01:46,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:01:46,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:46,040 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:01:46,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/ac8d6466a5e54a07ae43a3da91a6d33d is 50, key is test_row_0/A:col10/1733202105416/Put/seqid=0 2024-12-03T05:01:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:01:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:46,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:46,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742326_1502 (size=9857) 2024-12-03T05:01:46,073 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/ac8d6466a5e54a07ae43a3da91a6d33d 2024-12-03T05:01:46,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/a39b37c6c67443f7aae3c1dba3aa5f46 is 50, key is test_row_0/B:col10/1733202105416/Put/seqid=0 2024-12-03T05:01:46,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742327_1503 (size=9857) 2024-12-03T05:01:46,087 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/a39b37c6c67443f7aae3c1dba3aa5f46 2024-12-03T05:01:46,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/903d25932cee4de79deda3f2f3c2e2c1 is 50, key is test_row_0/C:col10/1733202105416/Put/seqid=0 2024-12-03T05:01:46,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742328_1504 (size=9857) 2024-12-03T05:01:46,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202166149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202166152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202166153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202166161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202166270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202166270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202166270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202166280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,321 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/dd4ddd95ec934e69993be0392357646c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dd4ddd95ec934e69993be0392357646c 2024-12-03T05:01:46,325 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/B of 1ef6c76f0cc0543400edbcde21d158fa into dd4ddd95ec934e69993be0392357646c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:46,325 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:46,325 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/B, priority=13, startTime=1733202105891; duration=0sec 2024-12-03T05:01:46,325 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:46,325 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:B 2024-12-03T05:01:46,325 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:01:46,327 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:01:46,327 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:01:46,327 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. because compaction request was cancelled 2024-12-03T05:01:46,327 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:C 2024-12-03T05:01:46,329 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/b364d9f7895442a087a767700b72b8d4 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b364d9f7895442a087a767700b72b8d4 2024-12-03T05:01:46,333 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/A of 1ef6c76f0cc0543400edbcde21d158fa into b364d9f7895442a087a767700b72b8d4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:46,333 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:46,333 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/A, priority=13, startTime=1733202105891; duration=0sec 2024-12-03T05:01:46,334 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:46,334 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:A 2024-12-03T05:01:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:01:46,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202166481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202166491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,498 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/903d25932cee4de79deda3f2f3c2e2c1 2024-12-03T05:01:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202166491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202166500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/ac8d6466a5e54a07ae43a3da91a6d33d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/ac8d6466a5e54a07ae43a3da91a6d33d 2024-12-03T05:01:46,546 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/ac8d6466a5e54a07ae43a3da91a6d33d, entries=100, sequenceid=280, filesize=9.6 K 2024-12-03T05:01:46,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/a39b37c6c67443f7aae3c1dba3aa5f46 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a39b37c6c67443f7aae3c1dba3aa5f46 2024-12-03T05:01:46,556 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a39b37c6c67443f7aae3c1dba3aa5f46, entries=100, sequenceid=280, filesize=9.6 K 2024-12-03T05:01:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/903d25932cee4de79deda3f2f3c2e2c1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/903d25932cee4de79deda3f2f3c2e2c1 2024-12-03T05:01:46,563 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/903d25932cee4de79deda3f2f3c2e2c1, entries=100, sequenceid=280, filesize=9.6 K 2024-12-03T05:01:46,564 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1ef6c76f0cc0543400edbcde21d158fa in 524ms, sequenceid=280, compaction requested=true 2024-12-03T05:01:46,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:46,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:46,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-03T05:01:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-03T05:01:46,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-03T05:01:46,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 833 msec 2024-12-03T05:01:46,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 837 msec 2024-12-03T05:01:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:46,807 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:46,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-03T05:01:46,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/051a66129ecf48989739ff63163a6b42 is 50, key is test_row_0/A:col10/1733202106151/Put/seqid=0 2024-12-03T05:01:46,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742329_1505 (size=14741) 2024-12-03T05:01:46,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202166820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202166829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202166830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202166830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:01:46,870 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:46,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-03T05:01:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T05:01:46,872 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:46,873 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:46,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:46,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202166941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202166941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202166941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:46,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202166950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:46,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T05:01:47,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T05:01:47,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:47,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202167151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202167151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202167151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:47,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T05:01:47,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:47,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202167170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T05:01:47,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/051a66129ecf48989739ff63163a6b42 2024-12-03T05:01:47,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/dd2c37a64be148118a6938f04321fd73 is 50, key is test_row_0/B:col10/1733202106151/Put/seqid=0 2024-12-03T05:01:47,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742330_1506 (size=12301) 2024-12-03T05:01:47,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T05:01:47,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:47,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202167460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:47,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202167460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:47,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202167470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:47,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T05:01:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202167491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T05:01:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:47,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/dd2c37a64be148118a6938f04321fd73 2024-12-03T05:01:47,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T05:01:47,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:47,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/5dcec101f7ca4c9cb42a694088dcff5c is 50, key is test_row_0/C:col10/1733202106151/Put/seqid=0 2024-12-03T05:01:47,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742331_1507 (size=12301) 2024-12-03T05:01:47,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T05:01:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T05:01:47,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:47,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:47,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:47,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202167980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:47,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202167980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:48,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202167990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T05:01:48,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202168011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:48,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/5dcec101f7ca4c9cb42a694088dcff5c 2024-12-03T05:01:48,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/051a66129ecf48989739ff63163a6b42 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/051a66129ecf48989739ff63163a6b42 2024-12-03T05:01:48,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/051a66129ecf48989739ff63163a6b42, entries=200, sequenceid=308, filesize=14.4 K 2024-12-03T05:01:48,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/dd2c37a64be148118a6938f04321fd73 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dd2c37a64be148118a6938f04321fd73 2024-12-03T05:01:48,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dd2c37a64be148118a6938f04321fd73, entries=150, sequenceid=308, filesize=12.0 K 2024-12-03T05:01:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/5dcec101f7ca4c9cb42a694088dcff5c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/5dcec101f7ca4c9cb42a694088dcff5c 2024-12-03T05:01:48,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/5dcec101f7ca4c9cb42a694088dcff5c, entries=150, sequenceid=308, filesize=12.0 K 2024-12-03T05:01:48,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 1ef6c76f0cc0543400edbcde21d158fa in 1279ms, sequenceid=308, compaction requested=true 2024-12-03T05:01:48,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:48,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:48,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:48,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:48,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:48,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:48,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:48,087 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:48,087 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:48,087 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37411 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:48,087 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:48,087 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/A is initiating minor compaction (all files) 2024-12-03T05:01:48,087 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/B is initiating minor compaction (all files) 2024-12-03T05:01:48,087 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/A in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:48,087 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/B in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:48,088 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b364d9f7895442a087a767700b72b8d4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/ac8d6466a5e54a07ae43a3da91a6d33d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/051a66129ecf48989739ff63163a6b42] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=36.5 K 2024-12-03T05:01:48,088 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dd4ddd95ec934e69993be0392357646c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a39b37c6c67443f7aae3c1dba3aa5f46, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dd2c37a64be148118a6938f04321fd73] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=34.2 K 2024-12-03T05:01:48,088 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting dd4ddd95ec934e69993be0392357646c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733202105258 2024-12-03T05:01:48,088 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b364d9f7895442a087a767700b72b8d4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733202105258 2024-12-03T05:01:48,088 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac8d6466a5e54a07ae43a3da91a6d33d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733202105416 2024-12-03T05:01:48,088 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting a39b37c6c67443f7aae3c1dba3aa5f46, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733202105416 2024-12-03T05:01:48,088 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 051a66129ecf48989739ff63163a6b42, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733202106133 2024-12-03T05:01:48,089 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting dd2c37a64be148118a6938f04321fd73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733202106133 2024-12-03T05:01:48,110 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#B#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:48,110 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#A#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:48,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T05:01:48,110 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/a25a3a895eed427282c843a3018c2525 is 50, key is test_row_0/A:col10/1733202106151/Put/seqid=0 2024-12-03T05:01:48,110 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/c277a5f460eb45d4859c2f6f540a9e96 is 50, key is test_row_0/B:col10/1733202106151/Put/seqid=0 2024-12-03T05:01:48,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:48,111 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:01:48,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742332_1508 (size=12915) 2024-12-03T05:01:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/8ad0ef9bdac9422a8744f8faa809eda3 is 50, key is test_row_0/A:col10/1733202106829/Put/seqid=0 2024-12-03T05:01:48,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742333_1509 (size=12915) 2024-12-03T05:01:48,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742334_1510 (size=12301) 2024-12-03T05:01:48,135 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/8ad0ef9bdac9422a8744f8faa809eda3 2024-12-03T05:01:48,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/26ba4b05292d4979807643e82958505f is 50, key is test_row_0/B:col10/1733202106829/Put/seqid=0 2024-12-03T05:01:48,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742335_1511 (size=12301) 2024-12-03T05:01:48,531 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/c277a5f460eb45d4859c2f6f540a9e96 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/c277a5f460eb45d4859c2f6f540a9e96 2024-12-03T05:01:48,535 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/a25a3a895eed427282c843a3018c2525 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a25a3a895eed427282c843a3018c2525 2024-12-03T05:01:48,536 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/B of 1ef6c76f0cc0543400edbcde21d158fa into c277a5f460eb45d4859c2f6f540a9e96(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:48,536 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:48,536 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/B, priority=13, startTime=1733202108086; duration=0sec 2024-12-03T05:01:48,536 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:48,536 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:B 2024-12-03T05:01:48,537 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:01:48,538 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:01:48,538 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/C is initiating minor compaction (all files) 2024-12-03T05:01:48,538 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/C in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:48,538 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/4fdcb90235df43b8b00da6a31428ae35, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/c833f881005c4e00aa17e1a3e2763de0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/903d25932cee4de79deda3f2f3c2e2c1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/5dcec101f7ca4c9cb42a694088dcff5c] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=46.0 K 2024-12-03T05:01:48,538 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/A of 1ef6c76f0cc0543400edbcde21d158fa into a25a3a895eed427282c843a3018c2525(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:48,538 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:48,538 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/A, priority=13, startTime=1733202108086; duration=0sec 2024-12-03T05:01:48,538 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fdcb90235df43b8b00da6a31428ae35, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733202104489 2024-12-03T05:01:48,539 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:48,539 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:A 2024-12-03T05:01:48,539 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c833f881005c4e00aa17e1a3e2763de0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733202105258 2024-12-03T05:01:48,539 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 903d25932cee4de79deda3f2f3c2e2c1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733202105416 2024-12-03T05:01:48,539 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dcec101f7ca4c9cb42a694088dcff5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733202106133 2024-12-03T05:01:48,546 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#C#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:48,547 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/10f105ad66c54561a6e0d8ab013df3db is 50, key is test_row_0/C:col10/1733202106151/Put/seqid=0 2024-12-03T05:01:48,549 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/26ba4b05292d4979807643e82958505f 2024-12-03T05:01:48,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742336_1512 (size=12881) 2024-12-03T05:01:48,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/aa0d21fab6514949bad5a6a641f1d7d0 is 50, key is test_row_0/C:col10/1733202106829/Put/seqid=0 2024-12-03T05:01:48,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742337_1513 (size=12301) 2024-12-03T05:01:48,964 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/10f105ad66c54561a6e0d8ab013df3db as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/10f105ad66c54561a6e0d8ab013df3db 2024-12-03T05:01:48,968 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/C of 1ef6c76f0cc0543400edbcde21d158fa into 10f105ad66c54561a6e0d8ab013df3db(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:48,968 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:48,968 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/C, priority=12, startTime=1733202108087; duration=0sec 2024-12-03T05:01:48,968 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:48,968 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:C 2024-12-03T05:01:48,977 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/aa0d21fab6514949bad5a6a641f1d7d0 2024-12-03T05:01:48,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/8ad0ef9bdac9422a8744f8faa809eda3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/8ad0ef9bdac9422a8744f8faa809eda3 2024-12-03T05:01:48,984 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/8ad0ef9bdac9422a8744f8faa809eda3, entries=150, sequenceid=319, filesize=12.0 K 2024-12-03T05:01:48,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/26ba4b05292d4979807643e82958505f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/26ba4b05292d4979807643e82958505f 2024-12-03T05:01:48,991 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/26ba4b05292d4979807643e82958505f, entries=150, sequenceid=319, filesize=12.0 K 2024-12-03T05:01:48,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/aa0d21fab6514949bad5a6a641f1d7d0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/aa0d21fab6514949bad5a6a641f1d7d0 2024-12-03T05:01:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:49,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:49,009 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/aa0d21fab6514949bad5a6a641f1d7d0, entries=150, sequenceid=319, filesize=12.0 K 2024-12-03T05:01:49,010 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=20.13 KB/20610 for 1ef6c76f0cc0543400edbcde21d158fa in 899ms, sequenceid=319, compaction requested=false 2024-12-03T05:01:49,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:49,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-03T05:01:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-03T05:01:49,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-03T05:01:49,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1390 sec 2024-12-03T05:01:49,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.1440 sec 2024-12-03T05:01:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:49,018 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:01:49,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-03T05:01:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T05:01:49,021 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:49,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:01:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-03T05:01:49,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/a217e81d284e47fba352be983007a095 is 50, key is test_row_0/A:col10/1733202109006/Put/seqid=0 2024-12-03T05:01:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:01:49,024 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:01:49,025 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:01:49,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:01:49,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742338_1514 (size=17181) 2024-12-03T05:01:49,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202169091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202169092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202169104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202169106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:01:49,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:01:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202169210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202169210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202169220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202169221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:01:49,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:49,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:01:49,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33850 deadline: 1733202169370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,378 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,34719,1733201977102 for 'test_row_1' in 1ef6c76f0cc0543400edbcde21d158fa of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18328 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T05:01:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202169430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202169431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/a217e81d284e47fba352be983007a095 2024-12-03T05:01:49,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202169440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202169440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/62b6d0fdb62944c19a9c7a381f94e416 is 50, key is test_row_0/B:col10/1733202109006/Put/seqid=0 2024-12-03T05:01:49,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742339_1515 (size=12301) 2024-12-03T05:01:49,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:01:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:01:49,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:01:49,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202169740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202169750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202169761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202169761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:49,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:01:49,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:49,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/62b6d0fdb62944c19a9c7a381f94e416 2024-12-03T05:01:49,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/b0221991cd6e49a8828c9ae3722c9a04 is 50, key is test_row_0/C:col10/1733202109006/Put/seqid=0 2024-12-03T05:01:49,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742340_1516 (size=12301) 2024-12-03T05:01:49,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:01:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:49,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:50,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:01:50,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:50,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:50,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:50,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:50,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:01:50,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:50,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33842 deadline: 1733202170253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:50,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:50,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:50,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:50,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:01:50,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:50,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:50,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:50,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:50,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:50,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:50,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33830 deadline: 1733202170271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:50,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:50,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:50,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:50,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:50,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33812 deadline: 1733202170280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:50,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:01:50,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1733202170280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 2024-12-03T05:01:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:01:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., hostname=32beccd76b35,34719,1733201977102, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:01:50,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:01:50,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/b0221991cd6e49a8828c9ae3722c9a04 2024-12-03T05:01:50,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/a217e81d284e47fba352be983007a095 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a217e81d284e47fba352be983007a095 2024-12-03T05:01:50,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a217e81d284e47fba352be983007a095, entries=250, sequenceid=333, filesize=16.8 K 2024-12-03T05:01:50,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/62b6d0fdb62944c19a9c7a381f94e416 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/62b6d0fdb62944c19a9c7a381f94e416 2024-12-03T05:01:50,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/62b6d0fdb62944c19a9c7a381f94e416, entries=150, sequenceid=333, filesize=12.0 K 2024-12-03T05:01:50,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/b0221991cd6e49a8828c9ae3722c9a04 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/b0221991cd6e49a8828c9ae3722c9a04 2024-12-03T05:01:50,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/b0221991cd6e49a8828c9ae3722c9a04, entries=150, sequenceid=333, filesize=12.0 K 2024-12-03T05:01:50,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1ef6c76f0cc0543400edbcde21d158fa in 1373ms, sequenceid=333, compaction requested=true 2024-12-03T05:01:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:01:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:50,391 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:01:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:50,391 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:50,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ef6c76f0cc0543400edbcde21d158fa:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:01:50,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:50,393 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42397 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:50,393 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:50,393 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/A is initiating minor compaction (all files) 2024-12-03T05:01:50,393 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/B is initiating minor compaction (all files) 2024-12-03T05:01:50,393 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/A in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:50,393 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/B in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:50,393 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a25a3a895eed427282c843a3018c2525, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/8ad0ef9bdac9422a8744f8faa809eda3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a217e81d284e47fba352be983007a095] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=41.4 K 2024-12-03T05:01:50,393 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/c277a5f460eb45d4859c2f6f540a9e96, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/26ba4b05292d4979807643e82958505f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/62b6d0fdb62944c19a9c7a381f94e416] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=36.6 K 2024-12-03T05:01:50,399 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c277a5f460eb45d4859c2f6f540a9e96, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733202106133 2024-12-03T05:01:50,399 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a25a3a895eed427282c843a3018c2525, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733202106133 2024-12-03T05:01:50,399 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad0ef9bdac9422a8744f8faa809eda3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733202106819 2024-12-03T05:01:50,399 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ba4b05292d4979807643e82958505f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733202106819 2024-12-03T05:01:50,401 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a217e81d284e47fba352be983007a095, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733202109006 2024-12-03T05:01:50,401 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 62b6d0fdb62944c19a9c7a381f94e416, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733202109006 2024-12-03T05:01:50,418 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#B#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:50,418 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/7c1b926ad3e5417bbc750020c7ea7ab4 is 50, key is test_row_0/B:col10/1733202109006/Put/seqid=0 2024-12-03T05:01:50,427 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:50,428 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/4c80104a70bc4522b668fcc3346231e1 is 50, key is test_row_0/A:col10/1733202109006/Put/seqid=0 2024-12-03T05:01:50,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:01:50,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:50,440 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-03T05:01:50,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742341_1517 (size=13017) 2024-12-03T05:01:50,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/655ac3650af84c82b4c3d052e9812804 is 50, key is test_row_0/A:col10/1733202109102/Put/seqid=0 2024-12-03T05:01:50,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742342_1518 (size=13017) 2024-12-03T05:01:50,525 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/7c1b926ad3e5417bbc750020c7ea7ab4 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7c1b926ad3e5417bbc750020c7ea7ab4 2024-12-03T05:01:50,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742343_1519 (size=12301) 2024-12-03T05:01:50,555 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/B of 1ef6c76f0cc0543400edbcde21d158fa into 7c1b926ad3e5417bbc750020c7ea7ab4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:50,555 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:50,555 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/B, priority=13, startTime=1733202110391; duration=0sec 2024-12-03T05:01:50,555 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:01:50,555 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:B 2024-12-03T05:01:50,555 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:01:50,557 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:01:50,557 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 1ef6c76f0cc0543400edbcde21d158fa/C is initiating minor compaction (all files) 2024-12-03T05:01:50,557 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ef6c76f0cc0543400edbcde21d158fa/C in TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:50,557 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/10f105ad66c54561a6e0d8ab013df3db, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/aa0d21fab6514949bad5a6a641f1d7d0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/b0221991cd6e49a8828c9ae3722c9a04] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp, totalSize=36.6 K 2024-12-03T05:01:50,561 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 10f105ad66c54561a6e0d8ab013df3db, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733202106133 2024-12-03T05:01:50,563 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting aa0d21fab6514949bad5a6a641f1d7d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733202106819 2024-12-03T05:01:50,565 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting b0221991cd6e49a8828c9ae3722c9a04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733202109006 2024-12-03T05:01:50,592 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ef6c76f0cc0543400edbcde21d158fa#C#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:01:50,593 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/ce8f205ec7414a64ab0233721d1ceee0 is 50, key is test_row_0/C:col10/1733202109006/Put/seqid=0 2024-12-03T05:01:50,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742344_1520 (size=12983) 2024-12-03T05:01:50,652 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/ce8f205ec7414a64ab0233721d1ceee0 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/ce8f205ec7414a64ab0233721d1ceee0 2024-12-03T05:01:50,689 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/C of 1ef6c76f0cc0543400edbcde21d158fa into ce8f205ec7414a64ab0233721d1ceee0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:50,689 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:50,689 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/C, priority=13, startTime=1733202110391; duration=0sec 2024-12-03T05:01:50,689 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:50,689 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:C 2024-12-03T05:01:50,795 INFO [Thread-2008 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2008. 2024-12-03T05:01:50,795 DEBUG [Thread-2008 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:50,795 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:50,796 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:50,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:50,798 INFO [Thread-2006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2006. 2024-12-03T05:01:50,798 DEBUG [Thread-2006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:50,798 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:50,801 INFO [Thread-2012 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2012. 2024-12-03T05:01:50,801 DEBUG [Thread-2012 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:50,801 DEBUG [Thread-2012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:50,801 DEBUG [Thread-2012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:50,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:50,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:50,809 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:50,810 INFO [Thread-2004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2004. 2024-12-03T05:01:50,810 DEBUG [Thread-2004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:50,810 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:50,810 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:50,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:50,814 INFO [Thread-2010 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2010. 2024-12-03T05:01:50,815 DEBUG [Thread-2010 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:50,815 DEBUG [Thread-2010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:50,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:50,815 DEBUG [Thread-2010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:50,890 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/4c80104a70bc4522b668fcc3346231e1 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/4c80104a70bc4522b668fcc3346231e1 2024-12-03T05:01:50,894 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ef6c76f0cc0543400edbcde21d158fa/A of 1ef6c76f0cc0543400edbcde21d158fa into 4c80104a70bc4522b668fcc3346231e1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:01:50,894 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:50,894 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., storeName=1ef6c76f0cc0543400edbcde21d158fa/A, priority=13, startTime=1733202110391; duration=0sec 2024-12-03T05:01:50,894 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:01:50,894 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ef6c76f0cc0543400edbcde21d158fa:A 2024-12-03T05:01:50,935 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/655ac3650af84c82b4c3d052e9812804 2024-12-03T05:01:50,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/cc7c6380a477425d8cffbb75f378fb04 is 50, key is test_row_0/B:col10/1733202109102/Put/seqid=0 2024-12-03T05:01:50,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742345_1521 (size=12301) 2024-12-03T05:01:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:01:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:51,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. as already flushing 2024-12-03T05:01:51,281 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-12-03T05:01:51,281 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:51,281 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:51,282 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:51,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:51,291 INFO [Thread-1997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1997. 2024-12-03T05:01:51,291 INFO [Thread-1995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1995. 2024-12-03T05:01:51,291 DEBUG [Thread-1997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:51,291 DEBUG [Thread-1995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:51,291 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:51,291 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:51,292 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:51,292 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:51,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:51,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:51,301 INFO [Thread-2001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2001. 2024-12-03T05:01:51,301 DEBUG [Thread-2001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:51,301 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:51,302 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:51,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:51,354 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/cc7c6380a477425d8cffbb75f378fb04 2024-12-03T05:01:51,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/f0cfa6ca42834b8cad44c59450f50a0b is 50, key is test_row_0/C:col10/1733202109102/Put/seqid=0 2024-12-03T05:01:51,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742346_1522 (size=12301) 2024-12-03T05:01:51,767 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/f0cfa6ca42834b8cad44c59450f50a0b 2024-12-03T05:01:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/655ac3650af84c82b4c3d052e9812804 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/655ac3650af84c82b4c3d052e9812804 2024-12-03T05:01:51,775 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/655ac3650af84c82b4c3d052e9812804, entries=150, sequenceid=358, filesize=12.0 K 2024-12-03T05:01:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/cc7c6380a477425d8cffbb75f378fb04 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/cc7c6380a477425d8cffbb75f378fb04 2024-12-03T05:01:51,778 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/cc7c6380a477425d8cffbb75f378fb04, entries=150, sequenceid=358, filesize=12.0 K 2024-12-03T05:01:51,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/f0cfa6ca42834b8cad44c59450f50a0b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/f0cfa6ca42834b8cad44c59450f50a0b 2024-12-03T05:01:51,782 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/f0cfa6ca42834b8cad44c59450f50a0b, entries=150, sequenceid=358, filesize=12.0 K 2024-12-03T05:01:51,783 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for 1ef6c76f0cc0543400edbcde21d158fa in 1342ms, sequenceid=358, compaction requested=false 2024-12-03T05:01:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 1ef6c76f0cc0543400edbcde21d158fa: 2024-12-03T05:01:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-03T05:01:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-03T05:01:51,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-03T05:01:51,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7590 sec 2024-12-03T05:01:51,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.7650 sec 2024-12-03T05:01:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:01:53,191 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:01:59,391 INFO [Thread-1999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1999. 2024-12-03T05:01:59,391 DEBUG [Thread-1999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:01:59,391 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:59,392 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:59,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1205 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3615 rows 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1225 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3670 rows 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1208 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3624 rows 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1220 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3650 rows 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1210 2024-12-03T05:01:59,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3630 rows 2024-12-03T05:01:59,392 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:01:59,392 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:01:59,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:59,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:01:59,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:01:59,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:01:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:01:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:01:59,397 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202119397"}]},"ts":"1733202119397"} 2024-12-03T05:01:59,399 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:01:59,399 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:01:59,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:01:59,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ef6c76f0cc0543400edbcde21d158fa, UNASSIGN}] 2024-12-03T05:01:59,401 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ef6c76f0cc0543400edbcde21d158fa, UNASSIGN 2024-12-03T05:01:59,402 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1ef6c76f0cc0543400edbcde21d158fa, regionState=CLOSING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:01:59,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ef6c76f0cc0543400edbcde21d158fa, UNASSIGN because future has completed 2024-12-03T05:01:59,404 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:01:59,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:01:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:01:59,556 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:01:59,556 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:01:59,556 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 1ef6c76f0cc0543400edbcde21d158fa, disabling compactions & flushes 2024-12-03T05:01:59,556 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:59,556 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:59,556 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. after waiting 0 ms 2024-12-03T05:01:59,556 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:01:59,556 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 1ef6c76f0cc0543400edbcde21d158fa 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-03T05:01:59,560 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/b97e1f32998849de981e642046810267 is 50, key is test_row_0/A:col10/1733202111290/Put/seqid=0 2024-12-03T05:01:59,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742347_1523 (size=12301) 2024-12-03T05:01:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:01:59,966 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/b97e1f32998849de981e642046810267 2024-12-03T05:01:59,974 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/a1547b7245494ed9a7448e39d976ab77 is 50, key is test_row_0/B:col10/1733202111290/Put/seqid=0 2024-12-03T05:01:59,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742348_1524 (size=12301) 2024-12-03T05:02:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:02:00,378 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/a1547b7245494ed9a7448e39d976ab77 2024-12-03T05:02:00,386 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/ed9b91d5b4e44873890afa2d4f6626f5 is 50, key is test_row_0/C:col10/1733202111290/Put/seqid=0 2024-12-03T05:02:00,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742349_1525 (size=12301) 2024-12-03T05:02:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:02:00,790 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/ed9b91d5b4e44873890afa2d4f6626f5 2024-12-03T05:02:00,793 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/A/b97e1f32998849de981e642046810267 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b97e1f32998849de981e642046810267 2024-12-03T05:02:00,795 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b97e1f32998849de981e642046810267, entries=150, sequenceid=369, filesize=12.0 K 2024-12-03T05:02:00,796 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/B/a1547b7245494ed9a7448e39d976ab77 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a1547b7245494ed9a7448e39d976ab77 2024-12-03T05:02:00,799 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a1547b7245494ed9a7448e39d976ab77, entries=150, sequenceid=369, filesize=12.0 K 2024-12-03T05:02:00,799 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/.tmp/C/ed9b91d5b4e44873890afa2d4f6626f5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/ed9b91d5b4e44873890afa2d4f6626f5 2024-12-03T05:02:00,802 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/ed9b91d5b4e44873890afa2d4f6626f5, entries=150, sequenceid=369, filesize=12.0 K 2024-12-03T05:02:00,802 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1ef6c76f0cc0543400edbcde21d158fa in 1246ms, sequenceid=369, compaction requested=true 2024-12-03T05:02:00,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/85255ea3cf0c4ce38189fa4fb17eaf3f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/9b0f0aae1dc44d4381ae3d458da1c839, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/572187e44f8f47148602814f8b03a70e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/7daaf89d9af940a7b2be4d3902ff7944, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/75cc714d2ff04b7ab05a61ab6fa00870, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/10f89e7353e84a23a1a5d9051329173d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3c1b94a551c441029debceb7ab5ecd2b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a6dc5f6ab4424d27b856b486a037d32b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/72cc77402e9b42389d5db0a40ddbd91d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/aa6ef64ab8254832b79f061992ab151f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b0bb828726834f2fb5a01be93aba74c4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/c82330d7a3a1493185348f941c1006c6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/16603526217e4d77a8814bd818f1575e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e5fb9a9dbb4d40c5ba450006f083666e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3705f483fb024129ab30702948a20af1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b364d9f7895442a087a767700b72b8d4, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/ac8d6466a5e54a07ae43a3da91a6d33d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/051a66129ecf48989739ff63163a6b42, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a25a3a895eed427282c843a3018c2525, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/8ad0ef9bdac9422a8744f8faa809eda3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a217e81d284e47fba352be983007a095] to archive 2024-12-03T05:02:00,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:02:00,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/85255ea3cf0c4ce38189fa4fb17eaf3f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/85255ea3cf0c4ce38189fa4fb17eaf3f 2024-12-03T05:02:00,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/9b0f0aae1dc44d4381ae3d458da1c839 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/9b0f0aae1dc44d4381ae3d458da1c839 2024-12-03T05:02:00,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/572187e44f8f47148602814f8b03a70e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/572187e44f8f47148602814f8b03a70e 2024-12-03T05:02:00,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/7daaf89d9af940a7b2be4d3902ff7944 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/7daaf89d9af940a7b2be4d3902ff7944 2024-12-03T05:02:00,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/75cc714d2ff04b7ab05a61ab6fa00870 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/75cc714d2ff04b7ab05a61ab6fa00870 2024-12-03T05:02:00,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/10f89e7353e84a23a1a5d9051329173d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/10f89e7353e84a23a1a5d9051329173d 2024-12-03T05:02:00,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3c1b94a551c441029debceb7ab5ecd2b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3c1b94a551c441029debceb7ab5ecd2b 2024-12-03T05:02:00,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a6dc5f6ab4424d27b856b486a037d32b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a6dc5f6ab4424d27b856b486a037d32b 2024-12-03T05:02:00,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/72cc77402e9b42389d5db0a40ddbd91d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/72cc77402e9b42389d5db0a40ddbd91d 2024-12-03T05:02:00,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/aa6ef64ab8254832b79f061992ab151f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/aa6ef64ab8254832b79f061992ab151f 2024-12-03T05:02:00,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b0bb828726834f2fb5a01be93aba74c4 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b0bb828726834f2fb5a01be93aba74c4 2024-12-03T05:02:00,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/c82330d7a3a1493185348f941c1006c6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/c82330d7a3a1493185348f941c1006c6 2024-12-03T05:02:00,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/16603526217e4d77a8814bd818f1575e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/16603526217e4d77a8814bd818f1575e 2024-12-03T05:02:00,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e5fb9a9dbb4d40c5ba450006f083666e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/e5fb9a9dbb4d40c5ba450006f083666e 2024-12-03T05:02:00,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3705f483fb024129ab30702948a20af1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/3705f483fb024129ab30702948a20af1 2024-12-03T05:02:00,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b364d9f7895442a087a767700b72b8d4 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b364d9f7895442a087a767700b72b8d4 2024-12-03T05:02:00,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/ac8d6466a5e54a07ae43a3da91a6d33d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/ac8d6466a5e54a07ae43a3da91a6d33d 2024-12-03T05:02:00,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/051a66129ecf48989739ff63163a6b42 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/051a66129ecf48989739ff63163a6b42 2024-12-03T05:02:00,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a25a3a895eed427282c843a3018c2525 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a25a3a895eed427282c843a3018c2525 2024-12-03T05:02:00,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/8ad0ef9bdac9422a8744f8faa809eda3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/8ad0ef9bdac9422a8744f8faa809eda3 2024-12-03T05:02:00,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a217e81d284e47fba352be983007a095 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/a217e81d284e47fba352be983007a095 2024-12-03T05:02:00,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/54c2cbd5bca946a5882eb6b8fce9370c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6e4dd7f83b6c4c67a5b364cdb729192d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/961583f9d8d9473bbbb8c6029ae4078c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/0f5941993c514903b925d34aa3fce91e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/363bdcad095a431090ddb21e1f317e8f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6221978d27064ff09f31feabcade032c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/cb9c55fc64da4800a09ddb44b8c9d9fc, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7c647e87845449e3a22e95d1fed3d359, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/1bbae731948e4df58302cfa0fb9f5ab9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6b23f30432234f1d987f56f6ab2f0c0f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dfd753c3571e4ff1b3d1ac6f201e4467, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/54f50240d9fe42dab09763dab39b2a70, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a8dcf3b806db4f9e830e9c4b4d77b320, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7818fdafb07a407fa4c209161f59e543, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dd4ddd95ec934e69993be0392357646c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/46acd714777842a88df76d2fafbd8470, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a39b37c6c67443f7aae3c1dba3aa5f46, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/c277a5f460eb45d4859c2f6f540a9e96, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dd2c37a64be148118a6938f04321fd73, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/26ba4b05292d4979807643e82958505f, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/62b6d0fdb62944c19a9c7a381f94e416] to archive 2024-12-03T05:02:00,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:02:00,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/54c2cbd5bca946a5882eb6b8fce9370c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/54c2cbd5bca946a5882eb6b8fce9370c 2024-12-03T05:02:00,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6e4dd7f83b6c4c67a5b364cdb729192d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6e4dd7f83b6c4c67a5b364cdb729192d 2024-12-03T05:02:00,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/961583f9d8d9473bbbb8c6029ae4078c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/961583f9d8d9473bbbb8c6029ae4078c 2024-12-03T05:02:00,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/0f5941993c514903b925d34aa3fce91e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/0f5941993c514903b925d34aa3fce91e 2024-12-03T05:02:00,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/363bdcad095a431090ddb21e1f317e8f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/363bdcad095a431090ddb21e1f317e8f 2024-12-03T05:02:00,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6221978d27064ff09f31feabcade032c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6221978d27064ff09f31feabcade032c 2024-12-03T05:02:00,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/cb9c55fc64da4800a09ddb44b8c9d9fc to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/cb9c55fc64da4800a09ddb44b8c9d9fc 2024-12-03T05:02:00,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7c647e87845449e3a22e95d1fed3d359 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7c647e87845449e3a22e95d1fed3d359 2024-12-03T05:02:00,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/1bbae731948e4df58302cfa0fb9f5ab9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/1bbae731948e4df58302cfa0fb9f5ab9 2024-12-03T05:02:00,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6b23f30432234f1d987f56f6ab2f0c0f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/6b23f30432234f1d987f56f6ab2f0c0f 2024-12-03T05:02:00,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dfd753c3571e4ff1b3d1ac6f201e4467 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dfd753c3571e4ff1b3d1ac6f201e4467 2024-12-03T05:02:00,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/54f50240d9fe42dab09763dab39b2a70 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/54f50240d9fe42dab09763dab39b2a70 2024-12-03T05:02:00,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a8dcf3b806db4f9e830e9c4b4d77b320 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a8dcf3b806db4f9e830e9c4b4d77b320 2024-12-03T05:02:00,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7818fdafb07a407fa4c209161f59e543 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7818fdafb07a407fa4c209161f59e543 2024-12-03T05:02:00,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dd4ddd95ec934e69993be0392357646c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dd4ddd95ec934e69993be0392357646c 2024-12-03T05:02:00,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/46acd714777842a88df76d2fafbd8470 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/46acd714777842a88df76d2fafbd8470 2024-12-03T05:02:00,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a39b37c6c67443f7aae3c1dba3aa5f46 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a39b37c6c67443f7aae3c1dba3aa5f46 2024-12-03T05:02:00,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/c277a5f460eb45d4859c2f6f540a9e96 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/c277a5f460eb45d4859c2f6f540a9e96 2024-12-03T05:02:00,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dd2c37a64be148118a6938f04321fd73 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/dd2c37a64be148118a6938f04321fd73 2024-12-03T05:02:00,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/26ba4b05292d4979807643e82958505f to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/26ba4b05292d4979807643e82958505f 2024-12-03T05:02:00,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/62b6d0fdb62944c19a9c7a381f94e416 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/62b6d0fdb62944c19a9c7a381f94e416 2024-12-03T05:02:00,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/6b4ae19a8a0043c6aebf7c120a18ed1e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/88a303b5f85843dfb2c10966d444743a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/f817747c957e4a0dbc24eb9e27c7ba28, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/90d4abec6ef34efba9d16dfae3134aa5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/7ab2d7928a8b4cfb90231d864f24139e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/eb9496890dbf418fb7176ee37bf10045, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/53d90e61f78c4f239721c048728645e0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/3e0250c3812d42c59122b09bb262125b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/e0342cbf799a4917a881ec436d4f20d3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/bbf33966815f49d3896ae38c5dbc7d51, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/77934522fbd84291a3ae8cd97ceadb8c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/54ef48af6e9c4960bd07936a374826df, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/4fdcb90235df43b8b00da6a31428ae35, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/44d0d80f4a4a49bfb7f4c0e422e9cb56, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/c833f881005c4e00aa17e1a3e2763de0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/903d25932cee4de79deda3f2f3c2e2c1, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/10f105ad66c54561a6e0d8ab013df3db, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/5dcec101f7ca4c9cb42a694088dcff5c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/aa0d21fab6514949bad5a6a641f1d7d0, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/b0221991cd6e49a8828c9ae3722c9a04] to archive 2024-12-03T05:02:00,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:02:00,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/6b4ae19a8a0043c6aebf7c120a18ed1e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/6b4ae19a8a0043c6aebf7c120a18ed1e 2024-12-03T05:02:00,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/88a303b5f85843dfb2c10966d444743a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/88a303b5f85843dfb2c10966d444743a 2024-12-03T05:02:00,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/f817747c957e4a0dbc24eb9e27c7ba28 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/f817747c957e4a0dbc24eb9e27c7ba28 2024-12-03T05:02:00,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/90d4abec6ef34efba9d16dfae3134aa5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/90d4abec6ef34efba9d16dfae3134aa5 2024-12-03T05:02:00,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/7ab2d7928a8b4cfb90231d864f24139e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/7ab2d7928a8b4cfb90231d864f24139e 2024-12-03T05:02:00,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/eb9496890dbf418fb7176ee37bf10045 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/eb9496890dbf418fb7176ee37bf10045 2024-12-03T05:02:00,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/53d90e61f78c4f239721c048728645e0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/53d90e61f78c4f239721c048728645e0 2024-12-03T05:02:00,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/3e0250c3812d42c59122b09bb262125b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/3e0250c3812d42c59122b09bb262125b 2024-12-03T05:02:00,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/e0342cbf799a4917a881ec436d4f20d3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/e0342cbf799a4917a881ec436d4f20d3 2024-12-03T05:02:00,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/bbf33966815f49d3896ae38c5dbc7d51 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/bbf33966815f49d3896ae38c5dbc7d51 2024-12-03T05:02:00,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/77934522fbd84291a3ae8cd97ceadb8c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/77934522fbd84291a3ae8cd97ceadb8c 2024-12-03T05:02:00,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/54ef48af6e9c4960bd07936a374826df to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/54ef48af6e9c4960bd07936a374826df 2024-12-03T05:02:00,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/4fdcb90235df43b8b00da6a31428ae35 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/4fdcb90235df43b8b00da6a31428ae35 2024-12-03T05:02:00,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/44d0d80f4a4a49bfb7f4c0e422e9cb56 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/44d0d80f4a4a49bfb7f4c0e422e9cb56 2024-12-03T05:02:00,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/c833f881005c4e00aa17e1a3e2763de0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/c833f881005c4e00aa17e1a3e2763de0 2024-12-03T05:02:00,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/903d25932cee4de79deda3f2f3c2e2c1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/903d25932cee4de79deda3f2f3c2e2c1 2024-12-03T05:02:00,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/10f105ad66c54561a6e0d8ab013df3db to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/10f105ad66c54561a6e0d8ab013df3db 2024-12-03T05:02:00,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/5dcec101f7ca4c9cb42a694088dcff5c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/5dcec101f7ca4c9cb42a694088dcff5c 2024-12-03T05:02:00,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/aa0d21fab6514949bad5a6a641f1d7d0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/aa0d21fab6514949bad5a6a641f1d7d0 2024-12-03T05:02:00,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/b0221991cd6e49a8828c9ae3722c9a04 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/b0221991cd6e49a8828c9ae3722c9a04 2024-12-03T05:02:00,887 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/recovered.edits/372.seqid, newMaxSeqId=372, maxSeqId=1 2024-12-03T05:02:00,888 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. 2024-12-03T05:02:00,888 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 1ef6c76f0cc0543400edbcde21d158fa: Waiting for close lock at 1733202119556Running coprocessor pre-close hooks at 1733202119556Disabling compacts and flushes for region at 1733202119556Disabling writes for close at 1733202119556Obtaining lock to block concurrent updates at 1733202119556Preparing flush snapshotting stores in 1ef6c76f0cc0543400edbcde21d158fa at 1733202119556Finished memstore snapshotting TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733202119557 (+1 ms)Flushing stores of TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa. at 1733202119557Flushing 1ef6c76f0cc0543400edbcde21d158fa/A: creating writer at 1733202119557Flushing 1ef6c76f0cc0543400edbcde21d158fa/A: appending metadata at 1733202119560 (+3 ms)Flushing 1ef6c76f0cc0543400edbcde21d158fa/A: closing flushed file at 1733202119560Flushing 1ef6c76f0cc0543400edbcde21d158fa/B: creating writer at 1733202119971 (+411 ms)Flushing 1ef6c76f0cc0543400edbcde21d158fa/B: appending metadata at 1733202119973 (+2 ms)Flushing 1ef6c76f0cc0543400edbcde21d158fa/B: closing flushed file at 1733202119973Flushing 1ef6c76f0cc0543400edbcde21d158fa/C: creating writer at 1733202120382 (+409 ms)Flushing 1ef6c76f0cc0543400edbcde21d158fa/C: appending metadata at 1733202120385 (+3 ms)Flushing 1ef6c76f0cc0543400edbcde21d158fa/C: closing flushed file at 1733202120385Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73955e76: reopening flushed file at 1733202120792 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63d159a4: reopening flushed file at 1733202120796 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@580b9418: reopening flushed file at 1733202120799 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1ef6c76f0cc0543400edbcde21d158fa in 1246ms, sequenceid=369, compaction requested=true at 1733202120803 (+4 ms)Writing region close event to WAL at 1733202120885 (+82 ms)Running coprocessor post-close hooks at 1733202120888 (+3 ms)Closed at 1733202120888 2024-12-03T05:02:00,890 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:02:00,890 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1ef6c76f0cc0543400edbcde21d158fa, regionState=CLOSED 2024-12-03T05:02:00,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:02:00,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-03T05:02:00,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 1ef6c76f0cc0543400edbcde21d158fa, server=32beccd76b35,34719,1733201977102 in 1.4880 sec 2024-12-03T05:02:00,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-12-03T05:02:00,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ef6c76f0cc0543400edbcde21d158fa, UNASSIGN in 1.4930 sec 2024-12-03T05:02:00,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-03T05:02:00,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4970 sec 2024-12-03T05:02:00,899 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202120899"}]},"ts":"1733202120899"} 2024-12-03T05:02:00,900 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T05:02:00,900 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T05:02:00,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5080 sec 2024-12-03T05:02:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:02:01,541 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:02:01,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T05:02:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:02:01,543 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:02:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T05:02:01,544 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:02:01,547 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:02:01,549 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C, FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/recovered.edits] 2024-12-03T05:02:01,551 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/4c80104a70bc4522b668fcc3346231e1 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/4c80104a70bc4522b668fcc3346231e1 2024-12-03T05:02:01,552 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/655ac3650af84c82b4c3d052e9812804 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/655ac3650af84c82b4c3d052e9812804 2024-12-03T05:02:01,553 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b97e1f32998849de981e642046810267 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/A/b97e1f32998849de981e642046810267 2024-12-03T05:02:01,554 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7c1b926ad3e5417bbc750020c7ea7ab4 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/7c1b926ad3e5417bbc750020c7ea7ab4 2024-12-03T05:02:01,555 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a1547b7245494ed9a7448e39d976ab77 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/a1547b7245494ed9a7448e39d976ab77 2024-12-03T05:02:01,556 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/cc7c6380a477425d8cffbb75f378fb04 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/B/cc7c6380a477425d8cffbb75f378fb04 2024-12-03T05:02:01,558 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/ce8f205ec7414a64ab0233721d1ceee0 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/ce8f205ec7414a64ab0233721d1ceee0 2024-12-03T05:02:01,559 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/ed9b91d5b4e44873890afa2d4f6626f5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/ed9b91d5b4e44873890afa2d4f6626f5 2024-12-03T05:02:01,560 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/f0cfa6ca42834b8cad44c59450f50a0b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/C/f0cfa6ca42834b8cad44c59450f50a0b 2024-12-03T05:02:01,562 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/recovered.edits/372.seqid to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa/recovered.edits/372.seqid 2024-12-03T05:02:01,562 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/1ef6c76f0cc0543400edbcde21d158fa 2024-12-03T05:02:01,563 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T05:02:01,564 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:02:01,566 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T05:02:01,567 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T05:02:01,568 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:02:01,568 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T05:02:01,569 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733202121568"}]},"ts":"9223372036854775807"} 2024-12-03T05:02:01,570 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T05:02:01,570 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1ef6c76f0cc0543400edbcde21d158fa, NAME => 'TestAcidGuarantees,,1733202088532.1ef6c76f0cc0543400edbcde21d158fa.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T05:02:01,570 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T05:02:01,570 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733202121570"}]},"ts":"9223372036854775807"} 2024-12-03T05:02:01,572 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T05:02:01,572 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:02:01,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-12-03T05:02:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T05:02:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T05:02:01,657 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:02:01,666 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 242), OpenFileDescriptor=441 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=714 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=4203 (was 3301) - AvailableMemoryMB LEAK? - 2024-12-03T05:02:01,678 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=714, ProcessCount=11, AvailableMemoryMB=4203 2024-12-03T05:02:01,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:02:01,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:02:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T05:02:01,682 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:02:01,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-12-03T05:02:01,682 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-03T05:02:01,683 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:02:01,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742350_1526 (size=959) 2024-12-03T05:02:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-03T05:02:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-03T05:02:02,092 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 29e602dd3df47e1bb4bce15079b1bb57, NAME => 'TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d 2024-12-03T05:02:02,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742351_1527 (size=53) 2024-12-03T05:02:02,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:02:02,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 29e602dd3df47e1bb4bce15079b1bb57, disabling compactions & flushes 2024-12-03T05:02:02,097 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:02,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:02,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. after waiting 0 ms 2024-12-03T05:02:02,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:02,097 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:02,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 29e602dd3df47e1bb4bce15079b1bb57: Waiting for close lock at 1733202122097Disabling compacts and flushes for region at 1733202122097Disabling writes for close at 1733202122097Writing region close event to WAL at 1733202122097Closed at 1733202122097 2024-12-03T05:02:02,098 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:02:02,099 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733202122098"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733202122098"}]},"ts":"1733202122098"} 2024-12-03T05:02:02,100 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:02:02,101 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:02:02,101 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202122101"}]},"ts":"1733202122101"} 2024-12-03T05:02:02,102 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T05:02:02,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, ASSIGN}] 2024-12-03T05:02:02,103 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, ASSIGN 2024-12-03T05:02:02,104 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, ASSIGN; state=OFFLINE, location=32beccd76b35,34719,1733201977102; forceNewPlan=false, retain=false 2024-12-03T05:02:02,254 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=29e602dd3df47e1bb4bce15079b1bb57, regionState=OPENING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:02:02,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, ASSIGN because future has completed 2024-12-03T05:02:02,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:02:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-03T05:02:02,411 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:02,411 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => 29e602dd3df47e1bb4bce15079b1bb57, NAME => 'TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:02:02,412 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,412 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:02:02,412 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,412 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,413 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,414 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29e602dd3df47e1bb4bce15079b1bb57 columnFamilyName A 2024-12-03T05:02:02,414 DEBUG [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:02,415 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(327): Store=29e602dd3df47e1bb4bce15079b1bb57/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:02:02,415 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,416 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29e602dd3df47e1bb4bce15079b1bb57 columnFamilyName B 2024-12-03T05:02:02,416 DEBUG [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:02,416 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(327): Store=29e602dd3df47e1bb4bce15079b1bb57/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:02:02,416 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,417 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29e602dd3df47e1bb4bce15079b1bb57 columnFamilyName C 2024-12-03T05:02:02,417 DEBUG [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:02,418 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(327): Store=29e602dd3df47e1bb4bce15079b1bb57/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:02:02,418 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,418 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,419 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,420 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,420 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,420 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:02:02,421 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,423 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:02:02,423 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened 29e602dd3df47e1bb4bce15079b1bb57; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72822742, jitterRate=0.08514341711997986}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:02:02,423 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:02,424 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for 29e602dd3df47e1bb4bce15079b1bb57: Running coprocessor pre-open hook at 1733202122412Writing region info on filesystem at 1733202122412Initializing all the Stores at 1733202122412Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202122413 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202122413Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202122413Cleaning up temporary data from old regions at 1733202122420 (+7 ms)Running coprocessor post-open hooks at 1733202122423 (+3 ms)Region opened successfully at 1733202122424 (+1 ms) 2024-12-03T05:02:02,425 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., pid=142, masterSystemTime=1733202122408 2024-12-03T05:02:02,427 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:02,427 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:02,427 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=29e602dd3df47e1bb4bce15079b1bb57, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:02:02,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:02:02,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-03T05:02:02,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 in 174 msec 2024-12-03T05:02:02,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-03T05:02:02,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, ASSIGN in 330 msec 2024-12-03T05:02:02,434 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:02:02,434 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202122434"}]},"ts":"1733202122434"} 2024-12-03T05:02:02,435 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T05:02:02,436 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:02:02,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 756 msec 2024-12-03T05:02:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-03T05:02:02,821 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:02:02,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66320673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:02,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:02:02,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:02:02,823 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:02:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:02:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:02:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38955848, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:02:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:02:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:02,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:02:02,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f57c5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:02,826 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:02:02,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:02:02,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:02:02,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:02:02,829 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:02:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,43423,1733201976038 2024-12-03T05:02:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@46262db2 2024-12-03T05:02:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:02:02,831 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:02:02,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:02:02,832 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:02:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T05:02:02,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742352_1528 (size=995) 2024-12-03T05:02:03,249 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-03T05:02:03,249 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-03T05:02:03,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:02:03,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, REOPEN/MOVE}] 2024-12-03T05:02:03,255 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, REOPEN/MOVE 2024-12-03T05:02:03,255 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=29e602dd3df47e1bb4bce15079b1bb57, regionState=CLOSING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, REOPEN/MOVE because future has completed 2024-12-03T05:02:03,257 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:02:03,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:02:03,409 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,409 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:02:03,409 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 29e602dd3df47e1bb4bce15079b1bb57, disabling compactions & flushes 2024-12-03T05:02:03,409 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:03,409 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:03,409 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. after waiting 0 ms 2024-12-03T05:02:03,409 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:03,413 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T05:02:03,418 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:03,418 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 29e602dd3df47e1bb4bce15079b1bb57: Waiting for close lock at 1733202123409Running coprocessor pre-close hooks at 1733202123409Disabling compacts and flushes for region at 1733202123409Disabling writes for close at 1733202123409Writing region close event to WAL at 1733202123410 (+1 ms)Running coprocessor post-close hooks at 1733202123417 (+7 ms)Closed at 1733202123417 2024-12-03T05:02:03,418 WARN [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: 29e602dd3df47e1bb4bce15079b1bb57 to self. 2024-12-03T05:02:03,420 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,421 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=29e602dd3df47e1bb4bce15079b1bb57, regionState=CLOSED 2024-12-03T05:02:03,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:02:03,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-03T05:02:03,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 in 166 msec 2024-12-03T05:02:03,426 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, REOPEN/MOVE; state=CLOSED, location=32beccd76b35,34719,1733201977102; forceNewPlan=false, retain=true 2024-12-03T05:02:03,577 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=29e602dd3df47e1bb4bce15079b1bb57, regionState=OPENING, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, REOPEN/MOVE because future has completed 2024-12-03T05:02:03,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102}] 2024-12-03T05:02:03,733 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:03,733 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => 29e602dd3df47e1bb4bce15079b1bb57, NAME => 'TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:02:03,734 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,734 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:02:03,734 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,734 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,735 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,736 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29e602dd3df47e1bb4bce15079b1bb57 columnFamilyName A 2024-12-03T05:02:03,737 DEBUG [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:03,737 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(327): Store=29e602dd3df47e1bb4bce15079b1bb57/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:02:03,737 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,738 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29e602dd3df47e1bb4bce15079b1bb57 columnFamilyName B 2024-12-03T05:02:03,738 DEBUG [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:03,738 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(327): Store=29e602dd3df47e1bb4bce15079b1bb57/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:02:03,739 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,739 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29e602dd3df47e1bb4bce15079b1bb57 columnFamilyName C 2024-12-03T05:02:03,739 DEBUG [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:03,739 INFO [StoreOpener-29e602dd3df47e1bb4bce15079b1bb57-1 {}] regionserver.HStore(327): Store=29e602dd3df47e1bb4bce15079b1bb57/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:02:03,739 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,740 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,741 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,742 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,742 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,742 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:02:03,743 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,744 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened 29e602dd3df47e1bb4bce15079b1bb57; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74201993, jitterRate=0.10569585859775543}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:02:03,744 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,745 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for 29e602dd3df47e1bb4bce15079b1bb57: Running coprocessor pre-open hook at 1733202123734Writing region info on filesystem at 1733202123734Initializing all the Stores at 1733202123734Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202123734Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202123735 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202123735Cleaning up temporary data from old regions at 1733202123742 (+7 ms)Running coprocessor post-open hooks at 1733202123744 (+2 ms)Region opened successfully at 1733202123745 (+1 ms) 2024-12-03T05:02:03,746 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., pid=147, masterSystemTime=1733202123731 2024-12-03T05:02:03,747 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:03,748 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:03,748 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=29e602dd3df47e1bb4bce15079b1bb57, regionState=OPEN, openSeqNum=5, regionLocation=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 because future has completed 2024-12-03T05:02:03,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-12-03T05:02:03,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 in 171 msec 2024-12-03T05:02:03,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-03T05:02:03,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, REOPEN/MOVE in 498 msec 2024-12-03T05:02:03,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-03T05:02:03,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-12-03T05:02:03,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 924 msec 2024-12-03T05:02:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-03T05:02:03,760 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:02:03,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:02:03,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:02:03,762 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:02:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:02:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:02:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c78cd47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:02:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:02:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:03,763 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:02:03,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc096e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:02:03,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:02:03,765 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:02:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:02:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:02:03,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c552a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:02:03,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:02:03,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:03,767 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:02:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128ac7bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:02:03,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:02:03,769 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:02:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:02:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:02:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3508f19e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:02:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:02:03,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:03,770 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:02:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32014c69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:02:03,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:02:03,772 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:02:03,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:02:03,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:02:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e283e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:02:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:02:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:03,773 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:02:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11fb1d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:02:03,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:02:03,775 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:02:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:02:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:02:03,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37e89898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:02:03,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:02:03,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:03,776 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:02:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d3abe92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:02:03,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:02:03,778 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:02:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:02:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:02:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d00879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:02:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:02:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:03,780 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:02:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7580163f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:02:03,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:02:03,781 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:02:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:02:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:02:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f774b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:02:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:02:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:03,783 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:02:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510078c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:02:03,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:02:03,785 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:02:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:02:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:02:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2821858f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:02:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:02:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:03,787 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:02:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7818c6e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:02:03,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:02:03,789 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:02:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:02:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:02:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16938c84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:02:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:02:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:03,790 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:02:03,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c795fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,791 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,43423,-1 for getting cluster id 2024-12-03T05:02:03,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:02:03,792 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f04a841-bf4e-4eda-85ec-19b4a03d3411' 2024-12-03T05:02:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:02:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f04a841-bf4e-4eda-85ec-19b4a03d3411" 2024-12-03T05:02:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4262767f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,43423,-1] 2024-12-03T05:02:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:02:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:03,793 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:02:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71bff62c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:02:03,794 DEBUG [Thread-2362 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:02:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:02:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:02:03,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:02:03,796 DEBUG [Thread-2360 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:02:03,796 DEBUG [Thread-2373 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:02:03,797 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:02:03,797 DEBUG [Thread-2356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:02:03,797 DEBUG [Thread-2360 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:02:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:02:03,797 DEBUG [Thread-2360 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:02:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:02:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:02:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-03T05:02:03,798 DEBUG [Thread-2358 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:02:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:02:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:02:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:02:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:02:03,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:02:03,799 DEBUG [Thread-2369 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:02:03,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:02:03,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:02:03,800 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:02:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:02:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:02:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:02:03,801 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:02:03,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:02:03,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:02:03,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:02:03,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:02:03,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:02:03,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:02:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:02:03,806 DEBUG [Thread-2367 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:02:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:02:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:02:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:02:03,808 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:02:03,809 DEBUG [Thread-2365 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:02:03,810 DEBUG [Thread-2365 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:02:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:02:03,810 DEBUG [Thread-2354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:02:03,810 DEBUG [Thread-2365 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:02:03,811 DEBUG [Thread-2354 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:02:03,811 DEBUG [Thread-2354 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:02:03,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:02:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:03,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-03T05:02:03,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:02:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:02:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:02:03,815 DEBUG [Thread-2371 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:02:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,34719,1733201977102, seqNum=-1] 2024-12-03T05:02:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:02:03,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:02:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5] 2024-12-03T05:02:03,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202183844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202183844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202183844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:03,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202183851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202183851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:03,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d1e39f32557c452bb00d67a58771eebb_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202123812/Put/seqid=0 2024-12-03T05:02:03,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742353_1529 (size=12154) 2024-12-03T05:02:03,892 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:03,896 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d1e39f32557c452bb00d67a58771eebb_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d1e39f32557c452bb00d67a58771eebb_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:03,897 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/067e9bd1bc9b47c6be084419ffd2543d, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:03,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/067e9bd1bc9b47c6be084419ffd2543d is 175, key is test_row_0/A:col10/1733202123812/Put/seqid=0 2024-12-03T05:02:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:02:03,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742354_1530 (size=30955) 2024-12-03T05:02:03,913 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/067e9bd1bc9b47c6be084419ffd2543d 2024-12-03T05:02:03,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/c653c64ef2454810878fbf66a0e7ce99 is 50, key is test_row_0/B:col10/1733202123812/Put/seqid=0 2024-12-03T05:02:03,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T05:02:03,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:03,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. as already flushing 2024-12-03T05:02:03,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:03,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 29e602dd3df47e1bb4bce15079b1bb57, NAME => 'TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:02:03,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 29e602dd3df47e1bb4bce15079b1bb57, NAME => 'TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:02:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29e602dd3df47e1bb4bce15079b1bb57, NAME => 'TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29e602dd3df47e1bb4bce15079b1bb57, NAME => 'TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:02:03,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202183963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:03,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:03,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202183963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202183963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:03,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202183969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202183969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:03,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742355_1531 (size=12001) 2024-12-03T05:02:03,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/c653c64ef2454810878fbf66a0e7ce99 2024-12-03T05:02:04,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/4f11147bbb1242d6aa070869c960c190 is 50, key is test_row_0/C:col10/1733202123812/Put/seqid=0 2024-12-03T05:02:04,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742356_1532 (size=12001) 2024-12-03T05:02:04,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/4f11147bbb1242d6aa070869c960c190 2024-12-03T05:02:04,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/067e9bd1bc9b47c6be084419ffd2543d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/067e9bd1bc9b47c6be084419ffd2543d 2024-12-03T05:02:04,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/067e9bd1bc9b47c6be084419ffd2543d, entries=150, sequenceid=18, filesize=30.2 K 2024-12-03T05:02:04,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/c653c64ef2454810878fbf66a0e7ce99 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/c653c64ef2454810878fbf66a0e7ce99 2024-12-03T05:02:04,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/c653c64ef2454810878fbf66a0e7ce99, entries=150, sequenceid=18, filesize=11.7 K 2024-12-03T05:02:04,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/4f11147bbb1242d6aa070869c960c190 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/4f11147bbb1242d6aa070869c960c190 2024-12-03T05:02:04,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/4f11147bbb1242d6aa070869c960c190, entries=150, sequenceid=18, filesize=11.7 K 2024-12-03T05:02:04,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 29e602dd3df47e1bb4bce15079b1bb57 in 253ms, sequenceid=18, compaction requested=false 2024-12-03T05:02:04,065 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T05:02:04,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:04,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T05:02:04,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:04,116 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-03T05:02:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:02:04,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f1e9db2a83ea4b90b6634e84ae591d7c_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202123835/Put/seqid=0 2024-12-03T05:02:04,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742357_1533 (size=12154) 2024-12-03T05:02:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:04,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. as already flushing 2024-12-03T05:02:04,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202184207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202184212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202184212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202184213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202184217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202184320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202184321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202184321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202184331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202184331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:02:04,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202184531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202184531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202184531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202184540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202184540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,548 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f1e9db2a83ea4b90b6634e84ae591d7c_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f1e9db2a83ea4b90b6634e84ae591d7c_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:04,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/14860d7df7134a85a52ef03126a5471e, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/14860d7df7134a85a52ef03126a5471e is 175, key is test_row_0/A:col10/1733202123835/Put/seqid=0 2024-12-03T05:02:04,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742358_1534 (size=30955) 2024-12-03T05:02:04,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202184840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202184840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202184840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202184851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202184851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:04,932 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:02:04,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:02:04,955 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/14860d7df7134a85a52ef03126a5471e 2024-12-03T05:02:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/939167f4e1b244c5b175fae4bef6b901 is 50, key is test_row_0/B:col10/1733202123835/Put/seqid=0 2024-12-03T05:02:04,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742359_1535 (size=12001) 2024-12-03T05:02:05,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202185351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:05,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202185351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:05,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202185353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:05,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202185360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:05,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202185360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:05,396 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/939167f4e1b244c5b175fae4bef6b901 2024-12-03T05:02:05,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/08dc2f90d47e4999b2a6a3ae36ea1fd6 is 50, key is test_row_0/C:col10/1733202123835/Put/seqid=0 2024-12-03T05:02:05,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742360_1536 (size=12001) 2024-12-03T05:02:05,423 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/08dc2f90d47e4999b2a6a3ae36ea1fd6 2024-12-03T05:02:05,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/14860d7df7134a85a52ef03126a5471e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/14860d7df7134a85a52ef03126a5471e 2024-12-03T05:02:05,431 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/14860d7df7134a85a52ef03126a5471e, entries=150, sequenceid=40, filesize=30.2 K 2024-12-03T05:02:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/939167f4e1b244c5b175fae4bef6b901 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/939167f4e1b244c5b175fae4bef6b901 2024-12-03T05:02:05,435 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/939167f4e1b244c5b175fae4bef6b901, entries=150, sequenceid=40, filesize=11.7 K 2024-12-03T05:02:05,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/08dc2f90d47e4999b2a6a3ae36ea1fd6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/08dc2f90d47e4999b2a6a3ae36ea1fd6 2024-12-03T05:02:05,439 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/08dc2f90d47e4999b2a6a3ae36ea1fd6, entries=150, sequenceid=40, filesize=11.7 K 2024-12-03T05:02:05,440 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 29e602dd3df47e1bb4bce15079b1bb57 in 1324ms, sequenceid=40, compaction requested=false 2024-12-03T05:02:05,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:05,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:05,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-03T05:02:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-03T05:02:05,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure.FlushRegionProcedure(167): procedure event for 149 is null, maybe the procedure is created when recovery 2024-12-03T05:02:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:02:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:06,379 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:06,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-03T05:02:06,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fdbf1c61ab564d3ea383f623400930a1_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202126373/Put/seqid=0 2024-12-03T05:02:06,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742361_1537 (size=12154) 2024-12-03T05:02:06,417 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:02:06,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202186451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202186453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202186454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202186454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202186455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202186560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202186570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202186571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202186571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202186571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202186770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202186780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202186780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202186781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202186781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:06,794 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:06,814 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fdbf1c61ab564d3ea383f623400930a1_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fdbf1c61ab564d3ea383f623400930a1_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:06,817 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/760d934e46ed44c68cd5e759f147cacb, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:06,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/760d934e46ed44c68cd5e759f147cacb is 175, key is test_row_0/A:col10/1733202126373/Put/seqid=0 2024-12-03T05:02:06,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742362_1538 (size=30955) 2024-12-03T05:02:07,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202187080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202187091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202187091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202187091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202187091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,266 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=58, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/760d934e46ed44c68cd5e759f147cacb 2024-12-03T05:02:07,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ed8508320d9945f0943e8400e77ab046 is 50, key is test_row_0/B:col10/1733202126373/Put/seqid=0 2024-12-03T05:02:07,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742363_1539 (size=12001) 2024-12-03T05:02:07,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ed8508320d9945f0943e8400e77ab046 2024-12-03T05:02:07,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/a1f32d2d993641879f87187a56b1a13b is 50, key is test_row_0/C:col10/1733202126373/Put/seqid=0 2024-12-03T05:02:07,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742364_1540 (size=12001) 2024-12-03T05:02:07,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/a1f32d2d993641879f87187a56b1a13b 2024-12-03T05:02:07,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/760d934e46ed44c68cd5e759f147cacb as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/760d934e46ed44c68cd5e759f147cacb 2024-12-03T05:02:07,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/760d934e46ed44c68cd5e759f147cacb, entries=150, sequenceid=58, filesize=30.2 K 2024-12-03T05:02:07,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ed8508320d9945f0943e8400e77ab046 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8508320d9945f0943e8400e77ab046 2024-12-03T05:02:07,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8508320d9945f0943e8400e77ab046, entries=150, sequenceid=58, filesize=11.7 K 2024-12-03T05:02:07,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/a1f32d2d993641879f87187a56b1a13b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/a1f32d2d993641879f87187a56b1a13b 2024-12-03T05:02:07,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/a1f32d2d993641879f87187a56b1a13b, entries=150, sequenceid=58, filesize=11.7 K 2024-12-03T05:02:07,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 29e602dd3df47e1bb4bce15079b1bb57 in 1114ms, sequenceid=58, compaction requested=true 2024-12-03T05:02:07,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:07,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:02:07,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:07,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:02:07,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:07,494 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:07,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:02:07,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:02:07,495 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:07,495 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/A is initiating minor compaction (all files) 2024-12-03T05:02:07,495 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/A in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:07,495 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/067e9bd1bc9b47c6be084419ffd2543d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/14860d7df7134a85a52ef03126a5471e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/760d934e46ed44c68cd5e759f147cacb] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=90.7 K 2024-12-03T05:02:07,495 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:07,495 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/067e9bd1bc9b47c6be084419ffd2543d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/14860d7df7134a85a52ef03126a5471e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/760d934e46ed44c68cd5e759f147cacb] 2024-12-03T05:02:07,495 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 067e9bd1bc9b47c6be084419ffd2543d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1733202123804 2024-12-03T05:02:07,496 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 14860d7df7134a85a52ef03126a5471e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733202123835 2024-12-03T05:02:07,496 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 760d934e46ed44c68cd5e759f147cacb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733202124212 2024-12-03T05:02:07,497 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:07,498 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:07,498 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/B is initiating minor compaction (all files) 2024-12-03T05:02:07,498 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/B in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:07,498 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/c653c64ef2454810878fbf66a0e7ce99, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/939167f4e1b244c5b175fae4bef6b901, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8508320d9945f0943e8400e77ab046] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=35.2 K 2024-12-03T05:02:07,499 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c653c64ef2454810878fbf66a0e7ce99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1733202123804 2024-12-03T05:02:07,501 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 939167f4e1b244c5b175fae4bef6b901, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733202123835 2024-12-03T05:02:07,501 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed8508320d9945f0943e8400e77ab046, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733202124212 2024-12-03T05:02:07,506 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:07,530 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120346602d5388ae484497a5faa5677d1b24_29e602dd3df47e1bb4bce15079b1bb57 store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:07,532 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120346602d5388ae484497a5faa5677d1b24_29e602dd3df47e1bb4bce15079b1bb57, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:07,532 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120346602d5388ae484497a5faa5677d1b24_29e602dd3df47e1bb4bce15079b1bb57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:07,551 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:07,552 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/8f27271243be4eb1a17ab18fa643b415 is 50, key is test_row_0/B:col10/1733202126373/Put/seqid=0 2024-12-03T05:02:07,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742365_1541 (size=4469) 2024-12-03T05:02:07,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742366_1542 (size=12104) 2024-12-03T05:02:07,577 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/8f27271243be4eb1a17ab18fa643b415 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/8f27271243be4eb1a17ab18fa643b415 2024-12-03T05:02:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:07,600 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:07,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-03T05:02:07,606 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/B of 29e602dd3df47e1bb4bce15079b1bb57 into 8f27271243be4eb1a17ab18fa643b415(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:07,606 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:07,606 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/B, priority=13, startTime=1733202127494; duration=0sec 2024-12-03T05:02:07,606 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:07,606 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:B 2024-12-03T05:02:07,606 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:07,608 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:07,608 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/C is initiating minor compaction (all files) 2024-12-03T05:02:07,608 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/C in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:07,608 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/4f11147bbb1242d6aa070869c960c190, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/08dc2f90d47e4999b2a6a3ae36ea1fd6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/a1f32d2d993641879f87187a56b1a13b] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=35.2 K 2024-12-03T05:02:07,608 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f11147bbb1242d6aa070869c960c190, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1733202123804 2024-12-03T05:02:07,609 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08dc2f90d47e4999b2a6a3ae36ea1fd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733202123835 2024-12-03T05:02:07,610 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f32d2d993641879f87187a56b1a13b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733202124212 2024-12-03T05:02:07,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203154da60a76f04055842eec7bce7247e5_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202126453/Put/seqid=0 2024-12-03T05:02:07,624 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:07,624 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/c9c394f3f37a479eb9610b7e4b160bbc is 50, key is test_row_0/C:col10/1733202126373/Put/seqid=0 2024-12-03T05:02:07,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742367_1543 (size=12154) 2024-12-03T05:02:07,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202187631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202187635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202187636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202187637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202187638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742368_1544 (size=12104) 2024-12-03T05:02:07,653 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/c9c394f3f37a479eb9610b7e4b160bbc as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/c9c394f3f37a479eb9610b7e4b160bbc 2024-12-03T05:02:07,659 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/C of 29e602dd3df47e1bb4bce15079b1bb57 into c9c394f3f37a479eb9610b7e4b160bbc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:07,659 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:07,659 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/C, priority=13, startTime=1733202127494; duration=0sec 2024-12-03T05:02:07,660 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:07,660 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:C 2024-12-03T05:02:07,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202187740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202187751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202187751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202187751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202187751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202187951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202187960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202187960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202187960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:02:07,973 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#A#compaction#455 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:07,973 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/8445e2b2bd704fa0af73c43bf3159f62 is 175, key is test_row_0/A:col10/1733202126373/Put/seqid=0 2024-12-03T05:02:07,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:07,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202187971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:07,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742369_1545 (size=31058) 2024-12-03T05:02:08,035 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:08,043 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203154da60a76f04055842eec7bce7247e5_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203154da60a76f04055842eec7bce7247e5_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:08,044 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/b95acd3ffe7d4b82a0bde8da3ec6dae2, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:08,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/b95acd3ffe7d4b82a0bde8da3ec6dae2 is 175, key is test_row_0/A:col10/1733202126453/Put/seqid=0 2024-12-03T05:02:08,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742370_1546 (size=30955) 2024-12-03T05:02:08,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202188261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:08,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202188270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:08,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202188270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:08,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202188272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:08,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202188280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:08,409 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/8445e2b2bd704fa0af73c43bf3159f62 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8445e2b2bd704fa0af73c43bf3159f62 2024-12-03T05:02:08,415 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/A of 29e602dd3df47e1bb4bce15079b1bb57 into 8445e2b2bd704fa0af73c43bf3159f62(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:08,415 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:08,415 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/A, priority=13, startTime=1733202127494; duration=0sec 2024-12-03T05:02:08,415 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:08,415 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:A 2024-12-03T05:02:08,469 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/b95acd3ffe7d4b82a0bde8da3ec6dae2 2024-12-03T05:02:08,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/9fb62e9532234f839285703a5ef8cc04 is 50, key is test_row_0/B:col10/1733202126453/Put/seqid=0 2024-12-03T05:02:08,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742371_1547 (size=12001) 2024-12-03T05:02:08,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:08,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202188773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:08,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202188781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:08,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202188782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:08,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202188791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:08,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202188791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:08,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:08,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/9fb62e9532234f839285703a5ef8cc04 2024-12-03T05:02:08,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/c899a42ff1594489bd9616f3c2727324 is 50, key is test_row_0/C:col10/1733202126453/Put/seqid=0 2024-12-03T05:02:08,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742372_1548 (size=12001) 2024-12-03T05:02:09,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/c899a42ff1594489bd9616f3c2727324 2024-12-03T05:02:09,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/b95acd3ffe7d4b82a0bde8da3ec6dae2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/b95acd3ffe7d4b82a0bde8da3ec6dae2 2024-12-03T05:02:09,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/b95acd3ffe7d4b82a0bde8da3ec6dae2, entries=150, sequenceid=78, filesize=30.2 K 2024-12-03T05:02:09,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/9fb62e9532234f839285703a5ef8cc04 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/9fb62e9532234f839285703a5ef8cc04 2024-12-03T05:02:09,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/9fb62e9532234f839285703a5ef8cc04, entries=150, sequenceid=78, filesize=11.7 K 2024-12-03T05:02:09,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/c899a42ff1594489bd9616f3c2727324 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/c899a42ff1594489bd9616f3c2727324 2024-12-03T05:02:09,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/c899a42ff1594489bd9616f3c2727324, entries=150, sequenceid=78, filesize=11.7 K 2024-12-03T05:02:09,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 29e602dd3df47e1bb4bce15079b1bb57 in 1746ms, sequenceid=78, compaction requested=false 2024-12-03T05:02:09,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:09,799 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:09,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-03T05:02:09,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e5308817dcf540ae9f48e5c8c9fa5e50_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202129792/Put/seqid=0 2024-12-03T05:02:09,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742373_1549 (size=17034) 2024-12-03T05:02:09,824 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:09,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202189817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:09,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202189819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:09,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e5308817dcf540ae9f48e5c8c9fa5e50_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e5308817dcf540ae9f48e5c8c9fa5e50_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:09,831 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/8ce22ae1945644c58ca268bb6ba73988, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:09,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202189825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:09,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/8ce22ae1945644c58ca268bb6ba73988 is 175, key is test_row_0/A:col10/1733202129792/Put/seqid=0 2024-12-03T05:02:09,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202189827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202189828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:09,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:09,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742374_1550 (size=48139) 2024-12-03T05:02:09,850 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/8ce22ae1945644c58ca268bb6ba73988 2024-12-03T05:02:09,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ce4521fbd15a4ca98f037c3a131bc434 is 50, key is test_row_0/B:col10/1733202129792/Put/seqid=0 2024-12-03T05:02:09,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742375_1551 (size=12001) 2024-12-03T05:02:09,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ce4521fbd15a4ca98f037c3a131bc434 2024-12-03T05:02:09,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/470f8680ded94675a246029c40f6e867 is 50, key is test_row_0/C:col10/1733202129792/Put/seqid=0 2024-12-03T05:02:09,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742376_1552 (size=12001) 2024-12-03T05:02:09,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/470f8680ded94675a246029c40f6e867 2024-12-03T05:02:09,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/8ce22ae1945644c58ca268bb6ba73988 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8ce22ae1945644c58ca268bb6ba73988 2024-12-03T05:02:09,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8ce22ae1945644c58ca268bb6ba73988, entries=250, sequenceid=100, filesize=47.0 K 2024-12-03T05:02:09,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ce4521fbd15a4ca98f037c3a131bc434 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ce4521fbd15a4ca98f037c3a131bc434 2024-12-03T05:02:09,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ce4521fbd15a4ca98f037c3a131bc434, entries=150, sequenceid=100, filesize=11.7 K 2024-12-03T05:02:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/470f8680ded94675a246029c40f6e867 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/470f8680ded94675a246029c40f6e867 2024-12-03T05:02:09,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/470f8680ded94675a246029c40f6e867, entries=150, sequenceid=100, filesize=11.7 K 2024-12-03T05:02:09,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 29e602dd3df47e1bb4bce15079b1bb57 in 118ms, sequenceid=100, compaction requested=true 2024-12-03T05:02:09,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:09,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:02:09,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:09,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:02:09,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:09,918 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:09,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:02:09,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:09,918 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:09,919 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/A is initiating minor compaction (all files) 2024-12-03T05:02:09,919 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/A in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:09,919 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8445e2b2bd704fa0af73c43bf3159f62, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/b95acd3ffe7d4b82a0bde8da3ec6dae2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8ce22ae1945644c58ca268bb6ba73988] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=107.6 K 2024-12-03T05:02:09,919 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:09,919 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8445e2b2bd704fa0af73c43bf3159f62, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/b95acd3ffe7d4b82a0bde8da3ec6dae2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8ce22ae1945644c58ca268bb6ba73988] 2024-12-03T05:02:09,919 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8445e2b2bd704fa0af73c43bf3159f62, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733202124212 2024-12-03T05:02:09,919 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b95acd3ffe7d4b82a0bde8da3ec6dae2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733202126450 2024-12-03T05:02:09,919 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:09,920 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ce22ae1945644c58ca268bb6ba73988, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733202127633 2024-12-03T05:02:09,920 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:09,920 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/B is initiating minor compaction (all files) 2024-12-03T05:02:09,920 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/B in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:09,921 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/8f27271243be4eb1a17ab18fa643b415, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/9fb62e9532234f839285703a5ef8cc04, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ce4521fbd15a4ca98f037c3a131bc434] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=35.3 K 2024-12-03T05:02:09,921 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f27271243be4eb1a17ab18fa643b415, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733202124212 2024-12-03T05:02:09,926 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb62e9532234f839285703a5ef8cc04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733202126450 2024-12-03T05:02:09,926 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ce4521fbd15a4ca98f037c3a131bc434, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733202127634 2024-12-03T05:02:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:09,934 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:09,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-03T05:02:09,938 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:09,954 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033a99d70776e54c9683ff3eb093f40614_29e602dd3df47e1bb4bce15079b1bb57 store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:09,957 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033a99d70776e54c9683ff3eb093f40614_29e602dd3df47e1bb4bce15079b1bb57, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:09,957 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033a99d70776e54c9683ff3eb093f40614_29e602dd3df47e1bb4bce15079b1bb57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:09,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202189961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:09,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202189962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:09,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:09,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202189962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:09,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202189963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:09,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202189965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:09,971 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:09,972 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/25c191fe96584c57a7cb2b84dea6269c is 50, key is test_row_0/B:col10/1733202129792/Put/seqid=0 2024-12-03T05:02:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:09,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120361a8bb02fd4c4ec6945b1903895abefc_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202129826/Put/seqid=0 2024-12-03T05:02:10,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742377_1553 (size=4469) 2024-12-03T05:02:10,018 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#A#compaction#464 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:10,019 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/3f6bef6b4f474d5e81e5068d71339c94 is 175, key is test_row_0/A:col10/1733202129792/Put/seqid=0 2024-12-03T05:02:10,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742378_1554 (size=12207) 2024-12-03T05:02:10,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742379_1555 (size=12154) 2024-12-03T05:02:10,032 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:10,035 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120361a8bb02fd4c4ec6945b1903895abefc_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120361a8bb02fd4c4ec6945b1903895abefc_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:10,036 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/25c191fe96584c57a7cb2b84dea6269c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/25c191fe96584c57a7cb2b84dea6269c 2024-12-03T05:02:10,037 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/531fe8fc82084f1fa9b2ead1c4a2c17c, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:10,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/531fe8fc82084f1fa9b2ead1c4a2c17c is 175, key is test_row_0/A:col10/1733202129826/Put/seqid=0 2024-12-03T05:02:10,041 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/B of 29e602dd3df47e1bb4bce15079b1bb57 into 25c191fe96584c57a7cb2b84dea6269c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:10,041 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:10,041 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/B, priority=13, startTime=1733202129918; duration=0sec 2024-12-03T05:02:10,042 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:10,042 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:B 2024-12-03T05:02:10,042 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:10,043 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:10,043 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/C is initiating minor compaction (all files) 2024-12-03T05:02:10,043 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/C in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:10,043 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/c9c394f3f37a479eb9610b7e4b160bbc, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/c899a42ff1594489bd9616f3c2727324, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/470f8680ded94675a246029c40f6e867] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=35.3 K 2024-12-03T05:02:10,043 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c9c394f3f37a479eb9610b7e4b160bbc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733202124212 2024-12-03T05:02:10,043 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c899a42ff1594489bd9616f3c2727324, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733202126450 2024-12-03T05:02:10,044 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 470f8680ded94675a246029c40f6e867, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733202127634 2024-12-03T05:02:10,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742381_1557 (size=30955) 2024-12-03T05:02:10,063 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:10,063 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/11a09988162441a19b2c2294a709cafb is 50, key is test_row_0/C:col10/1733202129792/Put/seqid=0 2024-12-03T05:02:10,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742380_1556 (size=31161) 2024-12-03T05:02:10,080 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/3f6bef6b4f474d5e81e5068d71339c94 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/3f6bef6b4f474d5e81e5068d71339c94 2024-12-03T05:02:10,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202190081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202190081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202190081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,085 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/A of 29e602dd3df47e1bb4bce15079b1bb57 into 3f6bef6b4f474d5e81e5068d71339c94(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:10,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,085 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:10,085 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/A, priority=13, startTime=1733202129918; duration=0sec 2024-12-03T05:02:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202190081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,085 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:10,085 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:A 2024-12-03T05:02:10,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202190081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742382_1558 (size=12207) 2024-12-03T05:02:10,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202190290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202190291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202190291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202190292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202190301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,459 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/531fe8fc82084f1fa9b2ead1c4a2c17c 2024-12-03T05:02:10,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/548ae559f36e44ddacab0c9ca53af91a is 50, key is test_row_0/B:col10/1733202129826/Put/seqid=0 2024-12-03T05:02:10,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742383_1559 (size=12001) 2024-12-03T05:02:10,511 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/11a09988162441a19b2c2294a709cafb as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/11a09988162441a19b2c2294a709cafb 2024-12-03T05:02:10,516 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/C of 29e602dd3df47e1bb4bce15079b1bb57 into 11a09988162441a19b2c2294a709cafb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:10,516 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:10,516 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/C, priority=13, startTime=1733202129918; duration=0sec 2024-12-03T05:02:10,516 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:10,516 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:C 2024-12-03T05:02:10,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202190606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202190606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202190606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202190607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202190614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:10,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/548ae559f36e44ddacab0c9ca53af91a 2024-12-03T05:02:10,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/3822f20f34404a84bee8ee428e2c440c is 50, key is test_row_0/C:col10/1733202129826/Put/seqid=0 2024-12-03T05:02:10,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742384_1560 (size=12001) 2024-12-03T05:02:10,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/3822f20f34404a84bee8ee428e2c440c 2024-12-03T05:02:10,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/531fe8fc82084f1fa9b2ead1c4a2c17c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/531fe8fc82084f1fa9b2ead1c4a2c17c 2024-12-03T05:02:10,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/531fe8fc82084f1fa9b2ead1c4a2c17c, entries=150, sequenceid=119, filesize=30.2 K 2024-12-03T05:02:10,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/548ae559f36e44ddacab0c9ca53af91a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/548ae559f36e44ddacab0c9ca53af91a 2024-12-03T05:02:10,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/548ae559f36e44ddacab0c9ca53af91a, entries=150, sequenceid=119, filesize=11.7 K 2024-12-03T05:02:10,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/3822f20f34404a84bee8ee428e2c440c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/3822f20f34404a84bee8ee428e2c440c 2024-12-03T05:02:10,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/3822f20f34404a84bee8ee428e2c440c, entries=150, sequenceid=119, filesize=11.7 K 2024-12-03T05:02:10,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 29e602dd3df47e1bb4bce15079b1bb57 in 986ms, sequenceid=119, compaction requested=false 2024-12-03T05:02:10,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:11,114 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:11,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-03T05:02:11,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c4fd4ea543b54545b34d47f86f273999_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202129960/Put/seqid=0 2024-12-03T05:02:11,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742385_1561 (size=14794) 2024-12-03T05:02:11,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202191182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202191182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202191183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202191186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202191189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202191300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202191301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202191301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202191301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202191301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202191511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202191511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202191511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202191511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202191511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,529 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:11,533 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c4fd4ea543b54545b34d47f86f273999_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c4fd4ea543b54545b34d47f86f273999_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:11,534 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/721549beb4964ef0a68e651a889041d2, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:11,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/721549beb4964ef0a68e651a889041d2 is 175, key is test_row_0/A:col10/1733202129960/Put/seqid=0 2024-12-03T05:02:11,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742386_1562 (size=39749) 2024-12-03T05:02:11,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/721549beb4964ef0a68e651a889041d2 2024-12-03T05:02:11,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/038f656d4773427f90413328238a5129 is 50, key is test_row_0/B:col10/1733202129960/Put/seqid=0 2024-12-03T05:02:11,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742387_1563 (size=12151) 2024-12-03T05:02:11,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202191820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202191821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202191821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202191821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202191821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:11,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/038f656d4773427f90413328238a5129 2024-12-03T05:02:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:02:12,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/715293dae036449aacff32558e235751 is 50, key is test_row_0/C:col10/1733202129960/Put/seqid=0 2024-12-03T05:02:12,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742388_1564 (size=12151) 2024-12-03T05:02:12,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/715293dae036449aacff32558e235751 2024-12-03T05:02:12,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/721549beb4964ef0a68e651a889041d2 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/721549beb4964ef0a68e651a889041d2 2024-12-03T05:02:12,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/721549beb4964ef0a68e651a889041d2, entries=200, sequenceid=140, filesize=38.8 K 2024-12-03T05:02:12,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/038f656d4773427f90413328238a5129 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/038f656d4773427f90413328238a5129 2024-12-03T05:02:12,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/038f656d4773427f90413328238a5129, entries=150, sequenceid=140, filesize=11.9 K 2024-12-03T05:02:12,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/715293dae036449aacff32558e235751 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/715293dae036449aacff32558e235751 2024-12-03T05:02:12,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/715293dae036449aacff32558e235751, entries=150, sequenceid=140, filesize=11.9 K 2024-12-03T05:02:12,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 29e602dd3df47e1bb4bce15079b1bb57 in 935ms, sequenceid=140, compaction requested=true 2024-12-03T05:02:12,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:12,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:02:12,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:12,050 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:12,050 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:12,051 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:12,051 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/B is initiating minor compaction (all files) 2024-12-03T05:02:12,051 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/B in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:12,051 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/25c191fe96584c57a7cb2b84dea6269c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/548ae559f36e44ddacab0c9ca53af91a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/038f656d4773427f90413328238a5129] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=35.5 K 2024-12-03T05:02:12,051 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:12,051 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/A is initiating minor compaction (all files) 2024-12-03T05:02:12,051 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/A in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:12,051 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/3f6bef6b4f474d5e81e5068d71339c94, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/531fe8fc82084f1fa9b2ead1c4a2c17c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/721549beb4964ef0a68e651a889041d2] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=99.5 K 2024-12-03T05:02:12,051 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:12,051 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/3f6bef6b4f474d5e81e5068d71339c94, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/531fe8fc82084f1fa9b2ead1c4a2c17c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/721549beb4964ef0a68e651a889041d2] 2024-12-03T05:02:12,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:02:12,052 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 25c191fe96584c57a7cb2b84dea6269c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733202127634 2024-12-03T05:02:12,052 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f6bef6b4f474d5e81e5068d71339c94, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733202127634 2024-12-03T05:02:12,052 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 548ae559f36e44ddacab0c9ca53af91a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733202129815 2024-12-03T05:02:12,052 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 531fe8fc82084f1fa9b2ead1c4a2c17c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733202129815 2024-12-03T05:02:12,053 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 038f656d4773427f90413328238a5129, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733202129960 2024-12-03T05:02:12,053 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721549beb4964ef0a68e651a889041d2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733202129960 2024-12-03T05:02:12,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:12,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:02:12,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:12,060 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:12,061 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#B#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:12,061 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/efba933283734c05895b86dcb58ff64b is 50, key is test_row_0/B:col10/1733202129960/Put/seqid=0 2024-12-03T05:02:12,066 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412039f0a4cdcfce641618c2413c823f92bc1_29e602dd3df47e1bb4bce15079b1bb57 store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:12,067 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412039f0a4cdcfce641618c2413c823f92bc1_29e602dd3df47e1bb4bce15079b1bb57, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:12,067 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039f0a4cdcfce641618c2413c823f92bc1_29e602dd3df47e1bb4bce15079b1bb57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:12,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742389_1565 (size=12459) 2024-12-03T05:02:12,082 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/efba933283734c05895b86dcb58ff64b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/efba933283734c05895b86dcb58ff64b 2024-12-03T05:02:12,086 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/B of 29e602dd3df47e1bb4bce15079b1bb57 into efba933283734c05895b86dcb58ff64b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:12,086 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:12,086 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/B, priority=13, startTime=1733202132050; duration=0sec 2024-12-03T05:02:12,086 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:12,086 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:B 2024-12-03T05:02:12,086 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:12,087 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:12,088 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/C is initiating minor compaction (all files) 2024-12-03T05:02:12,088 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/C in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:12,088 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/11a09988162441a19b2c2294a709cafb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/3822f20f34404a84bee8ee428e2c440c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/715293dae036449aacff32558e235751] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=35.5 K 2024-12-03T05:02:12,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742390_1566 (size=4469) 2024-12-03T05:02:12,089 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 11a09988162441a19b2c2294a709cafb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733202127634 2024-12-03T05:02:12,089 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3822f20f34404a84bee8ee428e2c440c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733202129815 2024-12-03T05:02:12,089 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 715293dae036449aacff32558e235751, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733202129960 2024-12-03T05:02:12,091 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#A#compaction#474 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:12,091 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/ed4284346c1e4942a8d0a083ead9321b is 175, key is test_row_0/A:col10/1733202129960/Put/seqid=0 2024-12-03T05:02:12,097 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#C#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:12,097 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/056f21187d9d4f749689f373d234bc26 is 50, key is test_row_0/C:col10/1733202129960/Put/seqid=0 2024-12-03T05:02:12,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742391_1567 (size=31413) 2024-12-03T05:02:12,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742392_1568 (size=12459) 2024-12-03T05:02:12,122 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/ed4284346c1e4942a8d0a083ead9321b as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ed4284346c1e4942a8d0a083ead9321b 2024-12-03T05:02:12,128 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/056f21187d9d4f749689f373d234bc26 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/056f21187d9d4f749689f373d234bc26 2024-12-03T05:02:12,130 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/A of 29e602dd3df47e1bb4bce15079b1bb57 into ed4284346c1e4942a8d0a083ead9321b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:12,130 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:12,131 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/A, priority=13, startTime=1733202132050; duration=0sec 2024-12-03T05:02:12,131 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:12,131 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:A 2024-12-03T05:02:12,135 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/C of 29e602dd3df47e1bb4bce15079b1bb57 into 056f21187d9d4f749689f373d234bc26(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:12,135 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:12,135 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/C, priority=13, startTime=1733202132054; duration=0sec 2024-12-03T05:02:12,135 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:12,135 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:C 2024-12-03T05:02:12,335 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:12,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-03T05:02:12,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba53e22472d24f58947b6290f2accc05_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202132335/Put/seqid=0 2024-12-03T05:02:12,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202192351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202192352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202192354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202192356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202192356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742393_1569 (size=12304) 2024-12-03T05:02:12,429 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:12,442 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba53e22472d24f58947b6290f2accc05_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba53e22472d24f58947b6290f2accc05_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:12,445 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/d5a3d19ab12e44cfaa10411c6e1e2c9e, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:12,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/d5a3d19ab12e44cfaa10411c6e1e2c9e is 175, key is test_row_0/A:col10/1733202132335/Put/seqid=0 2024-12-03T05:02:12,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202192461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202192465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202192470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202192471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202192471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742394_1570 (size=31105) 2024-12-03T05:02:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202192670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202192680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202192681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202192681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202192681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,887 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/d5a3d19ab12e44cfaa10411c6e1e2c9e 2024-12-03T05:02:12,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/af8fa4b554814e43b461d63b5b2f066d is 50, key is test_row_0/B:col10/1733202132335/Put/seqid=0 2024-12-03T05:02:12,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742395_1571 (size=12151) 2024-12-03T05:02:12,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202192980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202192991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202192991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202192991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:12,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202192991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:13,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/af8fa4b554814e43b461d63b5b2f066d 2024-12-03T05:02:13,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/bd62cce76edf48bb82c91346e2d23286 is 50, key is test_row_0/C:col10/1733202132335/Put/seqid=0 2024-12-03T05:02:13,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742396_1572 (size=12151) 2024-12-03T05:02:13,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202193500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:13,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202193500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:13,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202193500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:13,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202193501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:13,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202193501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:13,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/bd62cce76edf48bb82c91346e2d23286 2024-12-03T05:02:13,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/d5a3d19ab12e44cfaa10411c6e1e2c9e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5a3d19ab12e44cfaa10411c6e1e2c9e 2024-12-03T05:02:13,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5a3d19ab12e44cfaa10411c6e1e2c9e, entries=150, sequenceid=165, filesize=30.4 K 2024-12-03T05:02:13,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/af8fa4b554814e43b461d63b5b2f066d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/af8fa4b554814e43b461d63b5b2f066d 2024-12-03T05:02:13,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/af8fa4b554814e43b461d63b5b2f066d, entries=150, sequenceid=165, filesize=11.9 K 2024-12-03T05:02:13,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/bd62cce76edf48bb82c91346e2d23286 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bd62cce76edf48bb82c91346e2d23286 2024-12-03T05:02:13,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bd62cce76edf48bb82c91346e2d23286, entries=150, sequenceid=165, filesize=11.9 K 2024-12-03T05:02:13,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 29e602dd3df47e1bb4bce15079b1bb57 in 1452ms, sequenceid=165, compaction requested=false 2024-12-03T05:02:13,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:14,514 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:14,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-03T05:02:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:14,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030b99f65b38ea4d7c9db19cd5e4751abd_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202134512/Put/seqid=0 2024-12-03T05:02:14,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742397_1573 (size=14794) 2024-12-03T05:02:14,540 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:14,546 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030b99f65b38ea4d7c9db19cd5e4751abd_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030b99f65b38ea4d7c9db19cd5e4751abd_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:14,548 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/a2dee2182f764c1cac6974c628877d30, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:14,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/a2dee2182f764c1cac6974c628877d30 is 175, key is test_row_0/A:col10/1733202134512/Put/seqid=0 2024-12-03T05:02:14,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742398_1574 (size=39749) 2024-12-03T05:02:14,553 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/a2dee2182f764c1cac6974c628877d30 2024-12-03T05:02:14,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202194547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202194549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202194550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202194556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202194556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/e91906847a2844d6b7fe01b6e39a93c6 is 50, key is test_row_0/B:col10/1733202134512/Put/seqid=0 2024-12-03T05:02:14,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742399_1575 (size=12151) 2024-12-03T05:02:14,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/e91906847a2844d6b7fe01b6e39a93c6 2024-12-03T05:02:14,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/d309c7d44b2a4688a0ff474924e17b14 is 50, key is test_row_0/C:col10/1733202134512/Put/seqid=0 2024-12-03T05:02:14,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742400_1576 (size=12151) 2024-12-03T05:02:14,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/d309c7d44b2a4688a0ff474924e17b14 2024-12-03T05:02:14,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/a2dee2182f764c1cac6974c628877d30 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/a2dee2182f764c1cac6974c628877d30 2024-12-03T05:02:14,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/a2dee2182f764c1cac6974c628877d30, entries=200, sequenceid=180, filesize=38.8 K 2024-12-03T05:02:14,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/e91906847a2844d6b7fe01b6e39a93c6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/e91906847a2844d6b7fe01b6e39a93c6 2024-12-03T05:02:14,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202194661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202194661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202194661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202194671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202194671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/e91906847a2844d6b7fe01b6e39a93c6, entries=150, sequenceid=180, filesize=11.9 K 2024-12-03T05:02:14,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/d309c7d44b2a4688a0ff474924e17b14 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/d309c7d44b2a4688a0ff474924e17b14 2024-12-03T05:02:14,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/d309c7d44b2a4688a0ff474924e17b14, entries=150, sequenceid=180, filesize=11.9 K 2024-12-03T05:02:14,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 29e602dd3df47e1bb4bce15079b1bb57 in 175ms, sequenceid=180, compaction requested=true 2024-12-03T05:02:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:02:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:02:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:02:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:02:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T05:02:14,693 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:14,693 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:14,705 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:14,705 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/A is initiating minor compaction (all files) 2024-12-03T05:02:14,705 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/A in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:14,706 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ed4284346c1e4942a8d0a083ead9321b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5a3d19ab12e44cfaa10411c6e1e2c9e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/a2dee2182f764c1cac6974c628877d30] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=99.9 K 2024-12-03T05:02:14,706 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:14,706 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ed4284346c1e4942a8d0a083ead9321b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5a3d19ab12e44cfaa10411c6e1e2c9e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/a2dee2182f764c1cac6974c628877d30] 2024-12-03T05:02:14,706 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed4284346c1e4942a8d0a083ead9321b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733202129960 2024-12-03T05:02:14,707 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a3d19ab12e44cfaa10411c6e1e2c9e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733202131181 2024-12-03T05:02:14,708 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2dee2182f764c1cac6974c628877d30, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733202132349 2024-12-03T05:02:14,705 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:14,709 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/C is initiating minor compaction (all files) 2024-12-03T05:02:14,709 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/C in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:14,709 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/056f21187d9d4f749689f373d234bc26, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bd62cce76edf48bb82c91346e2d23286, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/d309c7d44b2a4688a0ff474924e17b14] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=35.9 K 2024-12-03T05:02:14,717 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 056f21187d9d4f749689f373d234bc26, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733202129960 2024-12-03T05:02:14,723 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting bd62cce76edf48bb82c91346e2d23286, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733202131181 2024-12-03T05:02:14,729 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting d309c7d44b2a4688a0ff474924e17b14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733202132352 2024-12-03T05:02:14,744 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:14,763 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#C#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:14,764 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/4cbc095f0aea46da854c54a577e4b40e is 50, key is test_row_0/C:col10/1733202134512/Put/seqid=0 2024-12-03T05:02:14,777 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120326af9097fb834687b943255011377083_29e602dd3df47e1bb4bce15079b1bb57 store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:14,780 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120326af9097fb834687b943255011377083_29e602dd3df47e1bb4bce15079b1bb57, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:14,780 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120326af9097fb834687b943255011377083_29e602dd3df47e1bb4bce15079b1bb57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:14,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742402_1578 (size=4469) 2024-12-03T05:02:14,794 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#A#compaction#482 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:14,795 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/23ff95c58c2941358d6cb4bb72c36f9a is 175, key is test_row_0/A:col10/1733202134512/Put/seqid=0 2024-12-03T05:02:14,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742401_1577 (size=12561) 2024-12-03T05:02:14,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742403_1579 (size=31515) 2024-12-03T05:02:14,807 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/23ff95c58c2941358d6cb4bb72c36f9a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/23ff95c58c2941358d6cb4bb72c36f9a 2024-12-03T05:02:14,812 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/A of 29e602dd3df47e1bb4bce15079b1bb57 into 23ff95c58c2941358d6cb4bb72c36f9a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:14,812 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:14,812 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/A, priority=13, startTime=1733202134690; duration=0sec 2024-12-03T05:02:14,812 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:14,812 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:A 2024-12-03T05:02:14,812 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:14,813 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:14,813 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/B is initiating minor compaction (all files) 2024-12-03T05:02:14,813 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/B in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:14,813 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/efba933283734c05895b86dcb58ff64b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/af8fa4b554814e43b461d63b5b2f066d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/e91906847a2844d6b7fe01b6e39a93c6] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=35.9 K 2024-12-03T05:02:14,813 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting efba933283734c05895b86dcb58ff64b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733202129960 2024-12-03T05:02:14,814 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting af8fa4b554814e43b461d63b5b2f066d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733202131181 2024-12-03T05:02:14,815 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e91906847a2844d6b7fe01b6e39a93c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733202132352 2024-12-03T05:02:14,820 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:14,821 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ce03b40448e5411999477a8868cb1a19 is 50, key is test_row_0/B:col10/1733202134512/Put/seqid=0 2024-12-03T05:02:14,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742404_1580 (size=12561) 2024-12-03T05:02:14,859 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ce03b40448e5411999477a8868cb1a19 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ce03b40448e5411999477a8868cb1a19 2024-12-03T05:02:14,870 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/B of 29e602dd3df47e1bb4bce15079b1bb57 into ce03b40448e5411999477a8868cb1a19(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:14,870 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:14,870 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/B, priority=13, startTime=1733202134690; duration=0sec 2024-12-03T05:02:14,871 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:14,871 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:B 2024-12-03T05:02:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:14,874 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:14,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-03T05:02:14,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b61d134d988c44a088935d749bcb656c_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202134552/Put/seqid=0 2024-12-03T05:02:14,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202194893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202194894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742405_1581 (size=12304) 2024-12-03T05:02:14,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202194896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202194897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202194895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:14,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:14,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:14,906 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:14,914 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b61d134d988c44a088935d749bcb656c_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b61d134d988c44a088935d749bcb656c_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:14,914 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/ac31f79fd3e549369ff598e057ab6420, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:14,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/ac31f79fd3e549369ff598e057ab6420 is 175, key is test_row_0/A:col10/1733202134552/Put/seqid=0 2024-12-03T05:02:14,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742406_1582 (size=31105) 2024-12-03T05:02:15,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202195011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202195012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202195012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202195012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202195012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202195221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202195221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202195221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202195222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202195222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,242 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/4cbc095f0aea46da854c54a577e4b40e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/4cbc095f0aea46da854c54a577e4b40e 2024-12-03T05:02:15,292 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/C of 29e602dd3df47e1bb4bce15079b1bb57 into 4cbc095f0aea46da854c54a577e4b40e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:15,292 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:15,293 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/C, priority=13, startTime=1733202134690; duration=0sec 2024-12-03T05:02:15,293 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:15,293 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:C 2024-12-03T05:02:15,367 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/ac31f79fd3e549369ff598e057ab6420 2024-12-03T05:02:15,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/397a8b4e13a14b678eda3794e9fa61f9 is 50, key is test_row_0/B:col10/1733202134552/Put/seqid=0 2024-12-03T05:02:15,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742407_1583 (size=12151) 2024-12-03T05:02:15,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202195531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202195531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202195531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202195532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202195540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:15,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/397a8b4e13a14b678eda3794e9fa61f9 2024-12-03T05:02:15,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/1e885c6c9f934e0695806562b78d045a is 50, key is test_row_0/C:col10/1733202134552/Put/seqid=0 2024-12-03T05:02:15,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742408_1584 (size=12151) 2024-12-03T05:02:16,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202196041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:16,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202196041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:16,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202196041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:16,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202196041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202196050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:16,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/1e885c6c9f934e0695806562b78d045a 2024-12-03T05:02:16,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/ac31f79fd3e549369ff598e057ab6420 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ac31f79fd3e549369ff598e057ab6420 2024-12-03T05:02:16,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ac31f79fd3e549369ff598e057ab6420, entries=150, sequenceid=204, filesize=30.4 K 2024-12-03T05:02:16,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/397a8b4e13a14b678eda3794e9fa61f9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/397a8b4e13a14b678eda3794e9fa61f9 2024-12-03T05:02:16,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/397a8b4e13a14b678eda3794e9fa61f9, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T05:02:16,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/1e885c6c9f934e0695806562b78d045a as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/1e885c6c9f934e0695806562b78d045a 2024-12-03T05:02:16,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/1e885c6c9f934e0695806562b78d045a, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T05:02:16,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 29e602dd3df47e1bb4bce15079b1bb57 in 1414ms, sequenceid=204, compaction requested=false 2024-12-03T05:02:16,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:17,054 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:17,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-03T05:02:17,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c79a6c437ff84be0900f3cd737545845_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202137053/Put/seqid=0 2024-12-03T05:02:17,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742409_1585 (size=12304) 2024-12-03T05:02:17,069 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:17,073 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c79a6c437ff84be0900f3cd737545845_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c79a6c437ff84be0900f3cd737545845_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:17,074 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/279c512a7ab84b8b9046d44da61ee53d, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:17,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/279c512a7ab84b8b9046d44da61ee53d is 175, key is test_row_0/A:col10/1733202137053/Put/seqid=0 2024-12-03T05:02:17,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202197073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202197075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202197079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202197080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202197080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742410_1586 (size=31105) 2024-12-03T05:02:17,109 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/279c512a7ab84b8b9046d44da61ee53d 2024-12-03T05:02:17,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/1f3a64f1412a470d8451a7310dea8c62 is 50, key is test_row_0/B:col10/1733202137053/Put/seqid=0 2024-12-03T05:02:17,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742411_1587 (size=12151) 2024-12-03T05:02:17,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202197191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202197191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202197191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202197191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202197193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202197401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202197401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202197401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202197401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202197405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/1f3a64f1412a470d8451a7310dea8c62 2024-12-03T05:02:17,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/73981986cfa748b2a4ca13096e067144 is 50, key is test_row_0/C:col10/1733202137053/Put/seqid=0 2024-12-03T05:02:17,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742412_1588 (size=12151) 2024-12-03T05:02:17,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202197710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202197711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202197711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202197711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202197720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:17,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/73981986cfa748b2a4ca13096e067144 2024-12-03T05:02:17,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/279c512a7ab84b8b9046d44da61ee53d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/279c512a7ab84b8b9046d44da61ee53d 2024-12-03T05:02:17,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/279c512a7ab84b8b9046d44da61ee53d, entries=150, sequenceid=222, filesize=30.4 K 2024-12-03T05:02:17,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/1f3a64f1412a470d8451a7310dea8c62 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/1f3a64f1412a470d8451a7310dea8c62 2024-12-03T05:02:17,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/1f3a64f1412a470d8451a7310dea8c62, entries=150, sequenceid=222, filesize=11.9 K 2024-12-03T05:02:17,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/73981986cfa748b2a4ca13096e067144 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/73981986cfa748b2a4ca13096e067144 2024-12-03T05:02:17,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/73981986cfa748b2a4ca13096e067144, entries=150, sequenceid=222, filesize=11.9 K 2024-12-03T05:02:18,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 29e602dd3df47e1bb4bce15079b1bb57 in 946ms, sequenceid=222, compaction requested=true 2024-12-03T05:02:18,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:18,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:02:18,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:18,000 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:18,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:02:18,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:18,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:02:18,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:02:18,000 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:18,001 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:18,001 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/A is initiating minor compaction (all files) 2024-12-03T05:02:18,001 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/A in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:18,001 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/23ff95c58c2941358d6cb4bb72c36f9a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ac31f79fd3e549369ff598e057ab6420, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/279c512a7ab84b8b9046d44da61ee53d] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=91.5 K 2024-12-03T05:02:18,001 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:18,001 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/23ff95c58c2941358d6cb4bb72c36f9a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ac31f79fd3e549369ff598e057ab6420, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/279c512a7ab84b8b9046d44da61ee53d] 2024-12-03T05:02:18,002 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ff95c58c2941358d6cb4bb72c36f9a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733202132352 2024-12-03T05:02:18,002 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac31f79fd3e549369ff598e057ab6420, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733202134544 2024-12-03T05:02:18,002 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:18,002 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/B is initiating minor compaction (all files) 2024-12-03T05:02:18,002 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/B in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:18,002 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ce03b40448e5411999477a8868cb1a19, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/397a8b4e13a14b678eda3794e9fa61f9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/1f3a64f1412a470d8451a7310dea8c62] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=36.0 K 2024-12-03T05:02:18,003 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 279c512a7ab84b8b9046d44da61ee53d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733202134884 2024-12-03T05:02:18,003 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ce03b40448e5411999477a8868cb1a19, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733202132352 2024-12-03T05:02:18,003 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 397a8b4e13a14b678eda3794e9fa61f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733202134544 2024-12-03T05:02:18,003 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3a64f1412a470d8451a7310dea8c62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733202134884 2024-12-03T05:02:18,010 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:18,012 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412037a6b48f4cb9041af9189e459fa510904_29e602dd3df47e1bb4bce15079b1bb57 store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:18,014 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412037a6b48f4cb9041af9189e459fa510904_29e602dd3df47e1bb4bce15079b1bb57, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:18,014 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037a6b48f4cb9041af9189e459fa510904_29e602dd3df47e1bb4bce15079b1bb57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:18,023 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:18,024 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/c7856e906b354ed99c0478785c25b47d is 50, key is test_row_0/B:col10/1733202137053/Put/seqid=0 2024-12-03T05:02:18,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742413_1589 (size=4469) 2024-12-03T05:02:18,028 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#A#compaction#491 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:18,029 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/6aa1f67948214d838ff7c8ba84cfbcb3 is 175, key is test_row_0/A:col10/1733202137053/Put/seqid=0 2024-12-03T05:02:18,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742414_1590 (size=31617) 2024-12-03T05:02:18,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742415_1591 (size=12663) 2024-12-03T05:02:18,045 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/6aa1f67948214d838ff7c8ba84cfbcb3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/6aa1f67948214d838ff7c8ba84cfbcb3 2024-12-03T05:02:18,055 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/c7856e906b354ed99c0478785c25b47d as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/c7856e906b354ed99c0478785c25b47d 2024-12-03T05:02:18,060 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/A of 29e602dd3df47e1bb4bce15079b1bb57 into 6aa1f67948214d838ff7c8ba84cfbcb3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:18,060 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:18,060 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/A, priority=13, startTime=1733202138000; duration=0sec 2024-12-03T05:02:18,060 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:18,060 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:A 2024-12-03T05:02:18,060 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:18,060 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/B of 29e602dd3df47e1bb4bce15079b1bb57 into c7856e906b354ed99c0478785c25b47d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:18,060 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:18,060 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/B, priority=13, startTime=1733202138000; duration=0sec 2024-12-03T05:02:18,060 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:18,060 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:B 2024-12-03T05:02:18,061 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:18,061 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/C is initiating minor compaction (all files) 2024-12-03T05:02:18,061 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/C in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:18,062 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/4cbc095f0aea46da854c54a577e4b40e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/1e885c6c9f934e0695806562b78d045a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/73981986cfa748b2a4ca13096e067144] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=36.0 K 2024-12-03T05:02:18,062 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cbc095f0aea46da854c54a577e4b40e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733202132352 2024-12-03T05:02:18,062 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e885c6c9f934e0695806562b78d045a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733202134544 2024-12-03T05:02:18,063 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73981986cfa748b2a4ca13096e067144, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733202134884 2024-12-03T05:02:18,069 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#C#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:18,070 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/d98c70967994477c951016992306b5ed is 50, key is test_row_0/C:col10/1733202137053/Put/seqid=0 2024-12-03T05:02:18,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742416_1592 (size=12663) 2024-12-03T05:02:18,096 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/d98c70967994477c951016992306b5ed as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/d98c70967994477c951016992306b5ed 2024-12-03T05:02:18,101 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/C of 29e602dd3df47e1bb4bce15079b1bb57 into d98c70967994477c951016992306b5ed(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:18,101 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:18,101 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/C, priority=13, startTime=1733202138000; duration=0sec 2024-12-03T05:02:18,101 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:18,101 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:C 2024-12-03T05:02:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:18,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:18,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-03T05:02:18,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c3e87f77ed9042b69628c53c11373c16_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202138222/Put/seqid=0 2024-12-03T05:02:18,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742417_1593 (size=12304) 2024-12-03T05:02:18,245 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:18,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202198243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,249 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c3e87f77ed9042b69628c53c11373c16_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c3e87f77ed9042b69628c53c11373c16_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,250 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/f885e90ce79243b7982b210cb53e8e03, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:18,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/f885e90ce79243b7982b210cb53e8e03 is 175, key is test_row_0/A:col10/1733202138222/Put/seqid=0 2024-12-03T05:02:18,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202198245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202198246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202198247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202198253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742418_1594 (size=31105) 2024-12-03T05:02:18,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202198350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202198361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202198361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202198361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202198361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202198560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202198571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202198571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202198571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202198571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,666 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/f885e90ce79243b7982b210cb53e8e03 2024-12-03T05:02:18,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/2db2ad763d5d414593c072e0a42c26bd is 50, key is test_row_0/B:col10/1733202138222/Put/seqid=0 2024-12-03T05:02:18,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742419_1595 (size=12151) 2024-12-03T05:02:18,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/2db2ad763d5d414593c072e0a42c26bd 2024-12-03T05:02:18,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/bcaca23dd98043c084f09c1b15fd5d73 is 50, key is test_row_0/C:col10/1733202138222/Put/seqid=0 2024-12-03T05:02:18,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742420_1596 (size=12151) 2024-12-03T05:02:18,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202198870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202198881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202198881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202198881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:18,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202198881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/bcaca23dd98043c084f09c1b15fd5d73 2024-12-03T05:02:19,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/f885e90ce79243b7982b210cb53e8e03 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/f885e90ce79243b7982b210cb53e8e03 2024-12-03T05:02:19,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/f885e90ce79243b7982b210cb53e8e03, entries=150, sequenceid=246, filesize=30.4 K 2024-12-03T05:02:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/2db2ad763d5d414593c072e0a42c26bd as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/2db2ad763d5d414593c072e0a42c26bd 2024-12-03T05:02:19,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/2db2ad763d5d414593c072e0a42c26bd, entries=150, sequenceid=246, filesize=11.9 K 2024-12-03T05:02:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/bcaca23dd98043c084f09c1b15fd5d73 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bcaca23dd98043c084f09c1b15fd5d73 2024-12-03T05:02:19,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bcaca23dd98043c084f09c1b15fd5d73, entries=150, sequenceid=246, filesize=11.9 K 2024-12-03T05:02:19,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 29e602dd3df47e1bb4bce15079b1bb57 in 905ms, sequenceid=246, compaction requested=false 2024-12-03T05:02:19,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:19,389 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:19,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-03T05:02:19,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c9e0b80e53f849a2ad363c7e82ee36f7_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202138245/Put/seqid=0 2024-12-03T05:02:19,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742421_1597 (size=12454) 2024-12-03T05:02:19,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202199414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202199419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202199420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202199420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202199421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202199530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202199531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202199531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202199531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202199531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202199740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202199741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202199741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202199741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202199744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:19,813 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:19,816 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c9e0b80e53f849a2ad363c7e82ee36f7_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c9e0b80e53f849a2ad363c7e82ee36f7_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:19,817 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/7a235189b31a4753b809137cd806c9dd, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:19,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/7a235189b31a4753b809137cd806c9dd is 175, key is test_row_0/A:col10/1733202138245/Put/seqid=0 2024-12-03T05:02:19,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742422_1598 (size=31255) 2024-12-03T05:02:20,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202200051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:20,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202200051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:20,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202200051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:20,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202200056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:20,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202200056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:20,222 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/7a235189b31a4753b809137cd806c9dd 2024-12-03T05:02:20,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ac2af4d1d41146f49a444dc099104b36 is 50, key is test_row_0/B:col10/1733202138245/Put/seqid=0 2024-12-03T05:02:20,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742423_1599 (size=12301) 2024-12-03T05:02:20,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202200561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:20,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202200561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:20,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202200570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202200570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:20,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202200570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:20,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ac2af4d1d41146f49a444dc099104b36 2024-12-03T05:02:20,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/cf3a83994d64400b98463d19dd1e7a26 is 50, key is test_row_0/C:col10/1733202138245/Put/seqid=0 2024-12-03T05:02:20,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742424_1600 (size=12301) 2024-12-03T05:02:20,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/cf3a83994d64400b98463d19dd1e7a26 2024-12-03T05:02:20,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/7a235189b31a4753b809137cd806c9dd as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/7a235189b31a4753b809137cd806c9dd 2024-12-03T05:02:20,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/7a235189b31a4753b809137cd806c9dd, entries=150, sequenceid=263, filesize=30.5 K 2024-12-03T05:02:20,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ac2af4d1d41146f49a444dc099104b36 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ac2af4d1d41146f49a444dc099104b36 2024-12-03T05:02:20,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ac2af4d1d41146f49a444dc099104b36, entries=150, sequenceid=263, filesize=12.0 K 2024-12-03T05:02:20,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/cf3a83994d64400b98463d19dd1e7a26 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/cf3a83994d64400b98463d19dd1e7a26 2024-12-03T05:02:20,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/cf3a83994d64400b98463d19dd1e7a26, entries=150, sequenceid=263, filesize=12.0 K 2024-12-03T05:02:20,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=120.76 KB/123660 for 29e602dd3df47e1bb4bce15079b1bb57 in 1353ms, sequenceid=263, compaction requested=true 2024-12-03T05:02:20,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:20,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:02:20,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:20,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:02:20,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:20,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:02:20,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:02:20,742 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:20,742 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:20,745 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93977 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:20,745 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/A is initiating minor compaction (all files) 2024-12-03T05:02:20,745 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/A in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:20,745 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/6aa1f67948214d838ff7c8ba84cfbcb3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/f885e90ce79243b7982b210cb53e8e03, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/7a235189b31a4753b809137cd806c9dd] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=91.8 K 2024-12-03T05:02:20,745 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:20,745 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/6aa1f67948214d838ff7c8ba84cfbcb3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/f885e90ce79243b7982b210cb53e8e03, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/7a235189b31a4753b809137cd806c9dd] 2024-12-03T05:02:20,745 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:20,746 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/B is initiating minor compaction (all files) 2024-12-03T05:02:20,746 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/B in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:20,746 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/c7856e906b354ed99c0478785c25b47d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/2db2ad763d5d414593c072e0a42c26bd, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ac2af4d1d41146f49a444dc099104b36] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=36.2 K 2024-12-03T05:02:20,746 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aa1f67948214d838ff7c8ba84cfbcb3, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733202134884 2024-12-03T05:02:20,747 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting c7856e906b354ed99c0478785c25b47d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733202134884 2024-12-03T05:02:20,747 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f885e90ce79243b7982b210cb53e8e03, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733202137071 2024-12-03T05:02:20,747 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2db2ad763d5d414593c072e0a42c26bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733202137071 2024-12-03T05:02:20,747 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a235189b31a4753b809137cd806c9dd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733202138242 2024-12-03T05:02:20,748 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ac2af4d1d41146f49a444dc099104b36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733202138242 2024-12-03T05:02:20,764 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:20,777 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:20,778 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/cf56cfc159674cdebbb2d15f8a4c85e5 is 50, key is test_row_0/B:col10/1733202138245/Put/seqid=0 2024-12-03T05:02:20,786 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203e5c9d9c77ac74edcb545bef653ce1c4c_29e602dd3df47e1bb4bce15079b1bb57 store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:20,787 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203e5c9d9c77ac74edcb545bef653ce1c4c_29e602dd3df47e1bb4bce15079b1bb57, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:20,787 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e5c9d9c77ac74edcb545bef653ce1c4c_29e602dd3df47e1bb4bce15079b1bb57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:20,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742425_1601 (size=12915) 2024-12-03T05:02:20,794 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/cf56cfc159674cdebbb2d15f8a4c85e5 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/cf56cfc159674cdebbb2d15f8a4c85e5 2024-12-03T05:02:20,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742426_1602 (size=4469) 2024-12-03T05:02:20,800 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/B of 29e602dd3df47e1bb4bce15079b1bb57 into cf56cfc159674cdebbb2d15f8a4c85e5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:20,800 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:20,800 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/B, priority=13, startTime=1733202140742; duration=0sec 2024-12-03T05:02:20,800 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:20,800 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:B 2024-12-03T05:02:20,801 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:20,803 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:20,803 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/C is initiating minor compaction (all files) 2024-12-03T05:02:20,803 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/C in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:20,803 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/d98c70967994477c951016992306b5ed, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bcaca23dd98043c084f09c1b15fd5d73, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/cf3a83994d64400b98463d19dd1e7a26] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=36.2 K 2024-12-03T05:02:20,804 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting d98c70967994477c951016992306b5ed, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733202134884 2024-12-03T05:02:20,804 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting bcaca23dd98043c084f09c1b15fd5d73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733202137071 2024-12-03T05:02:20,804 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting cf3a83994d64400b98463d19dd1e7a26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733202138242 2024-12-03T05:02:20,823 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#C#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:20,824 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/ccc1241f29254bd380c3b8fc14bf66cf is 50, key is test_row_0/C:col10/1733202138245/Put/seqid=0 2024-12-03T05:02:20,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742427_1603 (size=12915) 2024-12-03T05:02:20,879 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/ccc1241f29254bd380c3b8fc14bf66cf as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/ccc1241f29254bd380c3b8fc14bf66cf 2024-12-03T05:02:20,885 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/C of 29e602dd3df47e1bb4bce15079b1bb57 into ccc1241f29254bd380c3b8fc14bf66cf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:20,885 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:20,885 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/C, priority=13, startTime=1733202140742; duration=0sec 2024-12-03T05:02:20,885 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:20,885 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:C 2024-12-03T05:02:21,198 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#A#compaction#500 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:21,199 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/04970737c3104a6e972e36c095a72a0c is 175, key is test_row_0/A:col10/1733202138245/Put/seqid=0 2024-12-03T05:02:21,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742428_1604 (size=31869) 2024-12-03T05:02:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:21,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:21,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-03T05:02:21,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120384dce96bab9f4e87b3d1dcf9e7a8cbb2_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202139409/Put/seqid=0 2024-12-03T05:02:21,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202201597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202201602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202201606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202201606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202201606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742429_1605 (size=14994) 2024-12-03T05:02:21,640 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/04970737c3104a6e972e36c095a72a0c as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/04970737c3104a6e972e36c095a72a0c 2024-12-03T05:02:21,644 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/A of 29e602dd3df47e1bb4bce15079b1bb57 into 04970737c3104a6e972e36c095a72a0c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:21,644 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:21,644 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/A, priority=13, startTime=1733202140742; duration=0sec 2024-12-03T05:02:21,644 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:21,644 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:A 2024-12-03T05:02:21,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202201711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202201712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202201720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202201720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202201720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202201920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202201920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202201930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202201930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202201931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:02:22,017 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:22,038 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120384dce96bab9f4e87b3d1dcf9e7a8cbb2_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120384dce96bab9f4e87b3d1dcf9e7a8cbb2_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:22,042 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/adb880311b4e459382443ac4097628d3, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:22,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/adb880311b4e459382443ac4097628d3 is 175, key is test_row_0/A:col10/1733202139409/Put/seqid=0 2024-12-03T05:02:22,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742430_1606 (size=39949) 2024-12-03T05:02:22,075 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/adb880311b4e459382443ac4097628d3 2024-12-03T05:02:22,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ed8d47c66ab8408b8c26119cabba7215 is 50, key is test_row_0/B:col10/1733202139409/Put/seqid=0 2024-12-03T05:02:22,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742431_1607 (size=12301) 2024-12-03T05:02:22,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202202231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202202233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202202240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202202241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202202245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ed8d47c66ab8408b8c26119cabba7215 2024-12-03T05:02:22,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/77eed121393946869388c1d53a665870 is 50, key is test_row_0/C:col10/1733202139409/Put/seqid=0 2024-12-03T05:02:22,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742432_1608 (size=12301) 2024-12-03T05:02:22,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/77eed121393946869388c1d53a665870 2024-12-03T05:02:22,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/adb880311b4e459382443ac4097628d3 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/adb880311b4e459382443ac4097628d3 2024-12-03T05:02:22,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/adb880311b4e459382443ac4097628d3, entries=200, sequenceid=287, filesize=39.0 K 2024-12-03T05:02:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/ed8d47c66ab8408b8c26119cabba7215 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8d47c66ab8408b8c26119cabba7215 2024-12-03T05:02:22,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8d47c66ab8408b8c26119cabba7215, entries=150, sequenceid=287, filesize=12.0 K 2024-12-03T05:02:22,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/77eed121393946869388c1d53a665870 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/77eed121393946869388c1d53a665870 2024-12-03T05:02:22,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/77eed121393946869388c1d53a665870, entries=150, sequenceid=287, filesize=12.0 K 2024-12-03T05:02:22,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 29e602dd3df47e1bb4bce15079b1bb57 in 989ms, sequenceid=287, compaction requested=false 2024-12-03T05:02:22,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:22,743 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:22,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-03T05:02:22,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039c94206c169f414bbaceb3deeff56937_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202142741/Put/seqid=0 2024-12-03T05:02:22,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742433_1609 (size=14994) 2024-12-03T05:02:22,763 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:22,768 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039c94206c169f414bbaceb3deeff56937_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039c94206c169f414bbaceb3deeff56937_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:22,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/d5ca51978b0449c88f53461291613284, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:22,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/d5ca51978b0449c88f53461291613284 is 175, key is test_row_0/A:col10/1733202142741/Put/seqid=0 2024-12-03T05:02:22,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742434_1610 (size=39949) 2024-12-03T05:02:22,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202202774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202202776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202202776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202202777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202202778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202202891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202202891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202202892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202202892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202202892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:23,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202203100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:23,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202203100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:23,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202203111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:23,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202203111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:23,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202203111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:23,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/d5ca51978b0449c88f53461291613284 2024-12-03T05:02:23,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/49aa604be3724feb8f5a3b5c8d3bfe31 is 50, key is test_row_0/B:col10/1733202142741/Put/seqid=0 2024-12-03T05:02:23,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742435_1611 (size=12301) 2024-12-03T05:02:23,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40130 deadline: 1733202203410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:23,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40136 deadline: 1733202203411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:23,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40154 deadline: 1733202203422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:23,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:23,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40108 deadline: 1733202203422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:23,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:23,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T05:02:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40120 deadline: 1733202203422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 2024-12-03T05:02:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:02:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., hostname=32beccd76b35,34719,1733201977102, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:02:23,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/49aa604be3724feb8f5a3b5c8d3bfe31 2024-12-03T05:02:23,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/69c26ae284564601bf21ad52f2c888f9 is 50, key is test_row_0/C:col10/1733202142741/Put/seqid=0 2024-12-03T05:02:23,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742436_1612 (size=12301) 2024-12-03T05:02:23,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/69c26ae284564601bf21ad52f2c888f9 2024-12-03T05:02:23,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/d5ca51978b0449c88f53461291613284 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5ca51978b0449c88f53461291613284 2024-12-03T05:02:23,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5ca51978b0449c88f53461291613284, entries=200, sequenceid=303, filesize=39.0 K 2024-12-03T05:02:23,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/49aa604be3724feb8f5a3b5c8d3bfe31 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/49aa604be3724feb8f5a3b5c8d3bfe31 2024-12-03T05:02:23,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/49aa604be3724feb8f5a3b5c8d3bfe31, entries=150, sequenceid=303, filesize=12.0 K 2024-12-03T05:02:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/69c26ae284564601bf21ad52f2c888f9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/69c26ae284564601bf21ad52f2c888f9 2024-12-03T05:02:23,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/69c26ae284564601bf21ad52f2c888f9, entries=150, sequenceid=303, filesize=12.0 K 2024-12-03T05:02:23,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 29e602dd3df47e1bb4bce15079b1bb57 in 906ms, sequenceid=303, compaction requested=true 2024-12-03T05:02:23,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:23,649 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:23,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:02:23,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:23,649 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:23,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:02:23,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:23,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29e602dd3df47e1bb4bce15079b1bb57:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:02:23,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:23,651 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:23,651 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/B is initiating minor compaction (all files) 2024-12-03T05:02:23,651 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:23,651 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/B in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:23,651 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/A is initiating minor compaction (all files) 2024-12-03T05:02:23,651 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/cf56cfc159674cdebbb2d15f8a4c85e5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8d47c66ab8408b8c26119cabba7215, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/49aa604be3724feb8f5a3b5c8d3bfe31] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=36.6 K 2024-12-03T05:02:23,651 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/A in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:23,651 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/04970737c3104a6e972e36c095a72a0c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/adb880311b4e459382443ac4097628d3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5ca51978b0449c88f53461291613284] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=109.1 K 2024-12-03T05:02:23,651 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:23,651 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. files: [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/04970737c3104a6e972e36c095a72a0c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/adb880311b4e459382443ac4097628d3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5ca51978b0449c88f53461291613284] 2024-12-03T05:02:23,652 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting cf56cfc159674cdebbb2d15f8a4c85e5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733202138242 2024-12-03T05:02:23,652 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ed8d47c66ab8408b8c26119cabba7215, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733202139409 2024-12-03T05:02:23,652 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04970737c3104a6e972e36c095a72a0c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733202138242 2024-12-03T05:02:23,652 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 49aa604be3724feb8f5a3b5c8d3bfe31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733202141595 2024-12-03T05:02:23,652 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting adb880311b4e459382443ac4097628d3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733202139409 2024-12-03T05:02:23,653 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5ca51978b0449c88f53461291613284, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733202141595 2024-12-03T05:02:23,666 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:23,667 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:23,668 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/e5ec602c38a149bb98361aef77374bc6 is 50, key is test_row_0/B:col10/1733202142741/Put/seqid=0 2024-12-03T05:02:23,671 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203fb7459859c844970a76c1038a4d45592_29e602dd3df47e1bb4bce15079b1bb57 store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:23,673 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203fb7459859c844970a76c1038a4d45592_29e602dd3df47e1bb4bce15079b1bb57, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:23,673 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fb7459859c844970a76c1038a4d45592_29e602dd3df47e1bb4bce15079b1bb57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:23,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742437_1613 (size=13017) 2024-12-03T05:02:23,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742438_1614 (size=4469) 2024-12-03T05:02:23,711 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#A#compaction#510 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:23,712 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/a6bcfb3705f640ccbebf614d1e743b6f is 175, key is test_row_0/A:col10/1733202142741/Put/seqid=0 2024-12-03T05:02:23,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742439_1615 (size=31971) 2024-12-03T05:02:23,812 INFO [Thread-2371 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2371. 2024-12-03T05:02:23,812 INFO [Thread-2369 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2369. 2024-12-03T05:02:23,812 DEBUG [Thread-2371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:02:23,812 DEBUG [Thread-2369 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:02:23,812 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,812 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,812 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,812 INFO [Thread-2367 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2367. 2024-12-03T05:02:23,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:02:23,812 DEBUG [Thread-2367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:02:23,812 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,812 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,812 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:02:23,813 INFO [Thread-2373 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2373. 2024-12-03T05:02:23,813 DEBUG [Thread-2373 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:02:23,813 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,813 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,813 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:02:23,814 INFO [Thread-2365 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2365. 2024-12-03T05:02:23,814 DEBUG [Thread-2365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:02:23,814 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,814 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:02:23,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:02:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719 {}] regionserver.HRegion(8855): Flush requested on 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:23,922 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:02:23,922 INFO [Thread-2356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2356. 2024-12-03T05:02:23,922 DEBUG [Thread-2356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:02:23,922 INFO [Thread-2358 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2358. 2024-12-03T05:02:23,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-03T05:02:23,923 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,923 DEBUG [Thread-2358 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:02:23,923 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,923 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,923 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:02:23,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:02:23,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e44310757a174c748e4e6d5b5eeddd52_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_0/A:col10/1733202142776/Put/seqid=0 2024-12-03T05:02:23,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742440_1616 (size=12454) 2024-12-03T05:02:23,942 INFO [Thread-2354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2354. 2024-12-03T05:02:23,942 DEBUG [Thread-2354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:02:23,942 INFO [Thread-2360 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2360. 2024-12-03T05:02:23,942 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,942 INFO [Thread-2362 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2362. 2024-12-03T05:02:23,942 DEBUG [Thread-2360 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:02:23,942 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,942 DEBUG [Thread-2362 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:02:23,942 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,942 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,942 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:02:23,942 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:02:23,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:02:23,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:02:24,102 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/e5ec602c38a149bb98361aef77374bc6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/e5ec602c38a149bb98361aef77374bc6 2024-12-03T05:02:24,106 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/B of 29e602dd3df47e1bb4bce15079b1bb57 into e5ec602c38a149bb98361aef77374bc6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:24,106 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:24,106 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/B, priority=13, startTime=1733202143649; duration=0sec 2024-12-03T05:02:24,106 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:02:24,106 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:B 2024-12-03T05:02:24,106 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:02:24,107 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:02:24,107 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1541): 29e602dd3df47e1bb4bce15079b1bb57/C is initiating minor compaction (all files) 2024-12-03T05:02:24,107 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29e602dd3df47e1bb4bce15079b1bb57/C in TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:02:24,107 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/ccc1241f29254bd380c3b8fc14bf66cf, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/77eed121393946869388c1d53a665870, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/69c26ae284564601bf21ad52f2c888f9] into tmpdir=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp, totalSize=36.6 K 2024-12-03T05:02:24,108 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting ccc1241f29254bd380c3b8fc14bf66cf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733202138242 2024-12-03T05:02:24,108 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 77eed121393946869388c1d53a665870, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733202139409 2024-12-03T05:02:24,108 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c26ae284564601bf21ad52f2c888f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733202141595 2024-12-03T05:02:24,114 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29e602dd3df47e1bb4bce15079b1bb57#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:02:24,114 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/f2590fb4ca4148b1adbadac7afd4b952 is 50, key is test_row_0/C:col10/1733202142741/Put/seqid=0 2024-12-03T05:02:24,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742441_1617 (size=13017) 2024-12-03T05:02:24,133 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/a6bcfb3705f640ccbebf614d1e743b6f as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/a6bcfb3705f640ccbebf614d1e743b6f 2024-12-03T05:02:24,137 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/A of 29e602dd3df47e1bb4bce15079b1bb57 into a6bcfb3705f640ccbebf614d1e743b6f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:24,137 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:24,137 INFO [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/A, priority=13, startTime=1733202143649; duration=0sec 2024-12-03T05:02:24,137 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:24,137 DEBUG [RS:0;32beccd76b35:34719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:A 2024-12-03T05:02:24,333 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:02:24,336 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e44310757a174c748e4e6d5b5eeddd52_29e602dd3df47e1bb4bce15079b1bb57 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e44310757a174c748e4e6d5b5eeddd52_29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:02:24,336 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/64629bcd03c84876a3b5249df500a32e, store: [table=TestAcidGuarantees family=A region=29e602dd3df47e1bb4bce15079b1bb57] 2024-12-03T05:02:24,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/64629bcd03c84876a3b5249df500a32e is 175, key is test_row_0/A:col10/1733202142776/Put/seqid=0 2024-12-03T05:02:24,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742442_1618 (size=31255) 2024-12-03T05:02:24,521 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/f2590fb4ca4148b1adbadac7afd4b952 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/f2590fb4ca4148b1adbadac7afd4b952 2024-12-03T05:02:24,525 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29e602dd3df47e1bb4bce15079b1bb57/C of 29e602dd3df47e1bb4bce15079b1bb57 into f2590fb4ca4148b1adbadac7afd4b952(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:02:24,525 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:24,525 INFO [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57., storeName=29e602dd3df47e1bb4bce15079b1bb57/C, priority=13, startTime=1733202143650; duration=0sec 2024-12-03T05:02:24,526 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:02:24,526 DEBUG [RS:0;32beccd76b35:34719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29e602dd3df47e1bb4bce15079b1bb57:C 2024-12-03T05:02:24,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/64629bcd03c84876a3b5249df500a32e 2024-12-03T05:02:24,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/0f261caadd024b3cba0ab6acaeae31e9 is 50, key is test_row_0/B:col10/1733202142776/Put/seqid=0 2024-12-03T05:02:24,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742443_1619 (size=12301) 2024-12-03T05:02:25,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/0f261caadd024b3cba0ab6acaeae31e9 2024-12-03T05:02:25,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/1c8b56e42a6f463e9f9788cf162eb4c6 is 50, key is test_row_0/C:col10/1733202142776/Put/seqid=0 2024-12-03T05:02:25,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742444_1620 (size=12301) 2024-12-03T05:02:25,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/1c8b56e42a6f463e9f9788cf162eb4c6 2024-12-03T05:02:25,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/A/64629bcd03c84876a3b5249df500a32e as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/64629bcd03c84876a3b5249df500a32e 2024-12-03T05:02:25,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/64629bcd03c84876a3b5249df500a32e, entries=150, sequenceid=326, filesize=30.5 K 2024-12-03T05:02:25,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/B/0f261caadd024b3cba0ab6acaeae31e9 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/0f261caadd024b3cba0ab6acaeae31e9 2024-12-03T05:02:25,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/0f261caadd024b3cba0ab6acaeae31e9, entries=150, sequenceid=326, filesize=12.0 K 2024-12-03T05:02:25,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/.tmp/C/1c8b56e42a6f463e9f9788cf162eb4c6 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/1c8b56e42a6f463e9f9788cf162eb4c6 2024-12-03T05:02:25,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/1c8b56e42a6f463e9f9788cf162eb4c6, entries=150, sequenceid=326, filesize=12.0 K 2024-12-03T05:02:25,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=20.13 KB/20610 for 29e602dd3df47e1bb4bce15079b1bb57 in 1657ms, sequenceid=326, compaction requested=false 2024-12-03T05:02:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29e602dd3df47e1bb4bce15079b1bb57: 2024-12-03T05:02:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:02:34,932 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:02:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:02:48,734 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:02:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:03:04,932 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:03:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:03:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:03:33,734 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:03:34,933 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:03:39,023 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/067e9bd1bc9b47c6be084419ffd2543d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/14860d7df7134a85a52ef03126a5471e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8445e2b2bd704fa0af73c43bf3159f62, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/760d934e46ed44c68cd5e759f147cacb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/b95acd3ffe7d4b82a0bde8da3ec6dae2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8ce22ae1945644c58ca268bb6ba73988, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/3f6bef6b4f474d5e81e5068d71339c94, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/531fe8fc82084f1fa9b2ead1c4a2c17c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/721549beb4964ef0a68e651a889041d2, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ed4284346c1e4942a8d0a083ead9321b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5a3d19ab12e44cfaa10411c6e1e2c9e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/a2dee2182f764c1cac6974c628877d30, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/23ff95c58c2941358d6cb4bb72c36f9a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ac31f79fd3e549369ff598e057ab6420, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/6aa1f67948214d838ff7c8ba84cfbcb3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/279c512a7ab84b8b9046d44da61ee53d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/f885e90ce79243b7982b210cb53e8e03, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/04970737c3104a6e972e36c095a72a0c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/7a235189b31a4753b809137cd806c9dd, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/adb880311b4e459382443ac4097628d3, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5ca51978b0449c88f53461291613284] to archive 2024-12-03T05:03:39,030 WARN [regionserver/32beccd76b35:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-12-03T05:03:39,033 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:03:39,053 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/067e9bd1bc9b47c6be084419ffd2543d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/067e9bd1bc9b47c6be084419ffd2543d 2024-12-03T05:03:39,062 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/14860d7df7134a85a52ef03126a5471e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/14860d7df7134a85a52ef03126a5471e 2024-12-03T05:03:39,066 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8445e2b2bd704fa0af73c43bf3159f62 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8445e2b2bd704fa0af73c43bf3159f62 2024-12-03T05:03:39,068 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/760d934e46ed44c68cd5e759f147cacb to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/760d934e46ed44c68cd5e759f147cacb 2024-12-03T05:03:39,070 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/b95acd3ffe7d4b82a0bde8da3ec6dae2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/b95acd3ffe7d4b82a0bde8da3ec6dae2 2024-12-03T05:03:39,071 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8ce22ae1945644c58ca268bb6ba73988 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/8ce22ae1945644c58ca268bb6ba73988 2024-12-03T05:03:39,072 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/3f6bef6b4f474d5e81e5068d71339c94 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/3f6bef6b4f474d5e81e5068d71339c94 2024-12-03T05:03:39,073 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/531fe8fc82084f1fa9b2ead1c4a2c17c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/531fe8fc82084f1fa9b2ead1c4a2c17c 2024-12-03T05:03:39,074 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/721549beb4964ef0a68e651a889041d2 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/721549beb4964ef0a68e651a889041d2 2024-12-03T05:03:39,075 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ed4284346c1e4942a8d0a083ead9321b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ed4284346c1e4942a8d0a083ead9321b 2024-12-03T05:03:39,076 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5a3d19ab12e44cfaa10411c6e1e2c9e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5a3d19ab12e44cfaa10411c6e1e2c9e 2024-12-03T05:03:39,077 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/a2dee2182f764c1cac6974c628877d30 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/a2dee2182f764c1cac6974c628877d30 2024-12-03T05:03:39,078 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/23ff95c58c2941358d6cb4bb72c36f9a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/23ff95c58c2941358d6cb4bb72c36f9a 2024-12-03T05:03:39,079 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ac31f79fd3e549369ff598e057ab6420 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/ac31f79fd3e549369ff598e057ab6420 2024-12-03T05:03:39,080 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/6aa1f67948214d838ff7c8ba84cfbcb3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/6aa1f67948214d838ff7c8ba84cfbcb3 2024-12-03T05:03:39,081 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/279c512a7ab84b8b9046d44da61ee53d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/279c512a7ab84b8b9046d44da61ee53d 2024-12-03T05:03:39,082 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/f885e90ce79243b7982b210cb53e8e03 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/f885e90ce79243b7982b210cb53e8e03 2024-12-03T05:03:39,083 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/04970737c3104a6e972e36c095a72a0c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/04970737c3104a6e972e36c095a72a0c 2024-12-03T05:03:39,084 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/7a235189b31a4753b809137cd806c9dd to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/7a235189b31a4753b809137cd806c9dd 2024-12-03T05:03:39,085 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/adb880311b4e459382443ac4097628d3 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/adb880311b4e459382443ac4097628d3 2024-12-03T05:03:39,086 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5ca51978b0449c88f53461291613284 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/A/d5ca51978b0449c88f53461291613284 2024-12-03T05:03:39,093 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/c653c64ef2454810878fbf66a0e7ce99, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/939167f4e1b244c5b175fae4bef6b901, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/8f27271243be4eb1a17ab18fa643b415, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8508320d9945f0943e8400e77ab046, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/9fb62e9532234f839285703a5ef8cc04, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/25c191fe96584c57a7cb2b84dea6269c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ce4521fbd15a4ca98f037c3a131bc434, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/548ae559f36e44ddacab0c9ca53af91a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/efba933283734c05895b86dcb58ff64b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/038f656d4773427f90413328238a5129, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/af8fa4b554814e43b461d63b5b2f066d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ce03b40448e5411999477a8868cb1a19, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/e91906847a2844d6b7fe01b6e39a93c6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/397a8b4e13a14b678eda3794e9fa61f9, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/c7856e906b354ed99c0478785c25b47d, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/1f3a64f1412a470d8451a7310dea8c62, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/2db2ad763d5d414593c072e0a42c26bd, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/cf56cfc159674cdebbb2d15f8a4c85e5, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ac2af4d1d41146f49a444dc099104b36, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8d47c66ab8408b8c26119cabba7215, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/49aa604be3724feb8f5a3b5c8d3bfe31] to archive 2024-12-03T05:03:39,094 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:03:39,096 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/c653c64ef2454810878fbf66a0e7ce99 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/c653c64ef2454810878fbf66a0e7ce99 2024-12-03T05:03:39,097 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/939167f4e1b244c5b175fae4bef6b901 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/939167f4e1b244c5b175fae4bef6b901 2024-12-03T05:03:39,098 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/8f27271243be4eb1a17ab18fa643b415 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/8f27271243be4eb1a17ab18fa643b415 2024-12-03T05:03:39,099 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8508320d9945f0943e8400e77ab046 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8508320d9945f0943e8400e77ab046 2024-12-03T05:03:39,100 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/9fb62e9532234f839285703a5ef8cc04 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/9fb62e9532234f839285703a5ef8cc04 2024-12-03T05:03:39,100 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/25c191fe96584c57a7cb2b84dea6269c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/25c191fe96584c57a7cb2b84dea6269c 2024-12-03T05:03:39,101 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ce4521fbd15a4ca98f037c3a131bc434 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ce4521fbd15a4ca98f037c3a131bc434 2024-12-03T05:03:39,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/548ae559f36e44ddacab0c9ca53af91a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/548ae559f36e44ddacab0c9ca53af91a 2024-12-03T05:03:39,103 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/efba933283734c05895b86dcb58ff64b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/efba933283734c05895b86dcb58ff64b 2024-12-03T05:03:39,104 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/038f656d4773427f90413328238a5129 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/038f656d4773427f90413328238a5129 2024-12-03T05:03:39,105 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/af8fa4b554814e43b461d63b5b2f066d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/af8fa4b554814e43b461d63b5b2f066d 2024-12-03T05:03:39,106 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ce03b40448e5411999477a8868cb1a19 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ce03b40448e5411999477a8868cb1a19 2024-12-03T05:03:39,107 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/e91906847a2844d6b7fe01b6e39a93c6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/e91906847a2844d6b7fe01b6e39a93c6 2024-12-03T05:03:39,108 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/397a8b4e13a14b678eda3794e9fa61f9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/397a8b4e13a14b678eda3794e9fa61f9 2024-12-03T05:03:39,108 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/c7856e906b354ed99c0478785c25b47d to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/c7856e906b354ed99c0478785c25b47d 2024-12-03T05:03:39,109 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/1f3a64f1412a470d8451a7310dea8c62 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/1f3a64f1412a470d8451a7310dea8c62 2024-12-03T05:03:39,110 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/2db2ad763d5d414593c072e0a42c26bd to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/2db2ad763d5d414593c072e0a42c26bd 2024-12-03T05:03:39,111 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/cf56cfc159674cdebbb2d15f8a4c85e5 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/cf56cfc159674cdebbb2d15f8a4c85e5 2024-12-03T05:03:39,112 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ac2af4d1d41146f49a444dc099104b36 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ac2af4d1d41146f49a444dc099104b36 2024-12-03T05:03:39,113 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8d47c66ab8408b8c26119cabba7215 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/ed8d47c66ab8408b8c26119cabba7215 2024-12-03T05:03:39,114 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/49aa604be3724feb8f5a3b5c8d3bfe31 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/B/49aa604be3724feb8f5a3b5c8d3bfe31 2024-12-03T05:03:39,116 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/4f11147bbb1242d6aa070869c960c190, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/08dc2f90d47e4999b2a6a3ae36ea1fd6, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/c9c394f3f37a479eb9610b7e4b160bbc, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/a1f32d2d993641879f87187a56b1a13b, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/c899a42ff1594489bd9616f3c2727324, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/11a09988162441a19b2c2294a709cafb, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/470f8680ded94675a246029c40f6e867, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/3822f20f34404a84bee8ee428e2c440c, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/056f21187d9d4f749689f373d234bc26, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/715293dae036449aacff32558e235751, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bd62cce76edf48bb82c91346e2d23286, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/4cbc095f0aea46da854c54a577e4b40e, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/d309c7d44b2a4688a0ff474924e17b14, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/1e885c6c9f934e0695806562b78d045a, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/d98c70967994477c951016992306b5ed, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/73981986cfa748b2a4ca13096e067144, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bcaca23dd98043c084f09c1b15fd5d73, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/ccc1241f29254bd380c3b8fc14bf66cf, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/cf3a83994d64400b98463d19dd1e7a26, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/77eed121393946869388c1d53a665870, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/69c26ae284564601bf21ad52f2c888f9] to archive 2024-12-03T05:03:39,117 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:03:39,118 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/4f11147bbb1242d6aa070869c960c190 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/4f11147bbb1242d6aa070869c960c190 2024-12-03T05:03:39,120 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/08dc2f90d47e4999b2a6a3ae36ea1fd6 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/08dc2f90d47e4999b2a6a3ae36ea1fd6 2024-12-03T05:03:39,121 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/c9c394f3f37a479eb9610b7e4b160bbc to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/c9c394f3f37a479eb9610b7e4b160bbc 2024-12-03T05:03:39,122 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/a1f32d2d993641879f87187a56b1a13b to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/a1f32d2d993641879f87187a56b1a13b 2024-12-03T05:03:39,123 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/c899a42ff1594489bd9616f3c2727324 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/c899a42ff1594489bd9616f3c2727324 2024-12-03T05:03:39,124 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/11a09988162441a19b2c2294a709cafb to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/11a09988162441a19b2c2294a709cafb 2024-12-03T05:03:39,124 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/470f8680ded94675a246029c40f6e867 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/470f8680ded94675a246029c40f6e867 2024-12-03T05:03:39,125 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/3822f20f34404a84bee8ee428e2c440c to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/3822f20f34404a84bee8ee428e2c440c 2024-12-03T05:03:39,126 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/056f21187d9d4f749689f373d234bc26 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/056f21187d9d4f749689f373d234bc26 2024-12-03T05:03:39,127 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/715293dae036449aacff32558e235751 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/715293dae036449aacff32558e235751 2024-12-03T05:03:39,128 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bd62cce76edf48bb82c91346e2d23286 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bd62cce76edf48bb82c91346e2d23286 2024-12-03T05:03:39,129 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/4cbc095f0aea46da854c54a577e4b40e to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/4cbc095f0aea46da854c54a577e4b40e 2024-12-03T05:03:39,130 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/d309c7d44b2a4688a0ff474924e17b14 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/d309c7d44b2a4688a0ff474924e17b14 2024-12-03T05:03:39,131 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/1e885c6c9f934e0695806562b78d045a to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/1e885c6c9f934e0695806562b78d045a 2024-12-03T05:03:39,131 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/d98c70967994477c951016992306b5ed to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/d98c70967994477c951016992306b5ed 2024-12-03T05:03:39,132 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/73981986cfa748b2a4ca13096e067144 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/73981986cfa748b2a4ca13096e067144 2024-12-03T05:03:39,133 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bcaca23dd98043c084f09c1b15fd5d73 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/bcaca23dd98043c084f09c1b15fd5d73 2024-12-03T05:03:39,134 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/ccc1241f29254bd380c3b8fc14bf66cf to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/ccc1241f29254bd380c3b8fc14bf66cf 2024-12-03T05:03:39,135 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/cf3a83994d64400b98463d19dd1e7a26 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/cf3a83994d64400b98463d19dd1e7a26 2024-12-03T05:03:39,136 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/77eed121393946869388c1d53a665870 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/77eed121393946869388c1d53a665870 2024-12-03T05:03:39,136 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/69c26ae284564601bf21ad52f2c888f9 to hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/archive/data/default/TestAcidGuarantees/29e602dd3df47e1bb4bce15079b1bb57/C/69c26ae284564601bf21ad52f2c888f9 2024-12-03T05:03:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:04:04,933 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:04:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:04:18,735 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:04:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:04:34,933 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:04:37,229 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=693.26 KB, freeSize=879.32 MB, max=880 MB, blockCount=6, accesses=531667, hits=524899, hitRatio=98.73%, , cachingAccesses=525891, cachingHits=524438, cachingHitsRatio=99.72%, evictions=29, evicted=0, evictedPerRun=0.0 2024-12-03T05:04:37,332 DEBUG [master/32beccd76b35:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=3, created chunk count=13, reused chunk count=372, reuseRatio=96.62% 2024-12-03T05:04:37,332 DEBUG [master/32beccd76b35:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-12-03T05:04:39,110 INFO [regionserver/32beccd76b35:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-12-03T05:04:40,020 INFO [regionserver/32beccd76b35:0.Chore.1 {}] regionserver.HRegionServer$PeriodicMemStoreFlusher(1763): MemstoreFlusherChore requesting flush of hbase:meta,,1.1588230740 because 1588230740/ns has an old edit so flush to free WALs after random delay 174364 ms 2024-12-03T05:04:40,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48923, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:04:40,058 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(219): Skipping table TestAcidGuarantees because normalization is disabled in its table properties and normalization is also disabled at table level by default 2024-12-03T05:04:40,061 DEBUG [master/32beccd76b35:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T05:04:40,062 DEBUG [master/32beccd76b35:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-03T05:04:40,063 INFO [master/32beccd76b35:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-03T05:04:40,063 DEBUG [master/32beccd76b35:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-03T05:04:40,065 DEBUG [master/32beccd76b35:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {32beccd76b35=0} racks are {/default-rack=0} 2024-12-03T05:04:40,075 DEBUG [master/32beccd76b35:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions 2024-12-03T05:04:40,075 DEBUG [master/32beccd76b35:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T05:04:40,075 INFO [master/32beccd76b35:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T05:04:40,075 DEBUG [master/32beccd76b35:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-03T05:04:40,078 INFO [master/32beccd76b35:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s) 2024-12-03T05:04:40,078 DEBUG [master/32beccd76b35:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-03T05:04:45,218 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:04:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:05:03,735 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:05:04,933 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:05:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:05:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:05:34,934 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:05:39,076 WARN [regionserver/32beccd76b35:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-03T05:05:48,735 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:05:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:06:04,934 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:06:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:06:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:06:33,736 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:06:34,934 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:06:53,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:07:04,934 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:07:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:07:18,736 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:07:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:07:34,384 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-03T05:07:34,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=18.69 KB heapSize=33.02 KB 2024-12-03T05:07:34,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/info/095d6faa107f40a1af738ea927c10db4 is 157, key is TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57./info:regioninfo/1733202123748/Put/seqid=0 2024-12-03T05:07:34,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742445_1621 (size=8192) 2024-12-03T05:07:34,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.95 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/info/095d6faa107f40a1af738ea927c10db4 2024-12-03T05:07:34,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/ns/794cf0abe0724f3a853ab5fd7b261146 is 93, key is TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45./ns:/1733202005274/DeleteFamily/seqid=0 2024-12-03T05:07:34,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742446_1622 (size=5812) 2024-12-03T05:07:34,934 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:07:35,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=519 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/ns/794cf0abe0724f3a853ab5fd7b261146 2024-12-03T05:07:35,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/rep_barrier/3c6dd0d792eb4f81974d255062b5caae is 102, key is TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45./rep_barrier:/1733202005274/DeleteFamily/seqid=0 2024-12-03T05:07:35,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742447_1623 (size=5914) 2024-12-03T05:07:35,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=490 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/rep_barrier/3c6dd0d792eb4f81974d255062b5caae 2024-12-03T05:07:35,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/table/86fb74bbba314c389a5aaa5abd6f1829 is 96, key is TestAcidGuarantees,,1733201980210.1fc1c9cce64fed6eabb56af3d145cb45./table:/1733202005274/DeleteFamily/seqid=0 2024-12-03T05:07:35,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742448_1624 (size=5941) 2024-12-03T05:07:36,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.76 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/table/86fb74bbba314c389a5aaa5abd6f1829 2024-12-03T05:07:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/info/095d6faa107f40a1af738ea927c10db4 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/info/095d6faa107f40a1af738ea927c10db4 2024-12-03T05:07:36,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/info/095d6faa107f40a1af738ea927c10db4, entries=25, sequenceid=83, filesize=8 K 2024-12-03T05:07:36,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/ns/794cf0abe0724f3a853ab5fd7b261146 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/ns/794cf0abe0724f3a853ab5fd7b261146 2024-12-03T05:07:36,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/ns/794cf0abe0724f3a853ab5fd7b261146, entries=7, sequenceid=83, filesize=5.7 K 2024-12-03T05:07:36,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/rep_barrier/3c6dd0d792eb4f81974d255062b5caae as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/rep_barrier/3c6dd0d792eb4f81974d255062b5caae 2024-12-03T05:07:36,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/rep_barrier/3c6dd0d792eb4f81974d255062b5caae, entries=5, sequenceid=83, filesize=5.8 K 2024-12-03T05:07:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/.tmp/table/86fb74bbba314c389a5aaa5abd6f1829 as hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/table/86fb74bbba314c389a5aaa5abd6f1829 2024-12-03T05:07:36,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/data/hbase/meta/1588230740/table/86fb74bbba314c389a5aaa5abd6f1829, entries=8, sequenceid=83, filesize=5.8 K 2024-12-03T05:07:36,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.69 KB/19139, heapSize ~32.95 KB/33744, currentSize=0 B/0 for 1588230740 in 1713ms, sequenceid=83, compaction requested=false 2024-12-03T05:07:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-12-03T05:07:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:07:54,702 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 25859 2024-12-03T05:08:03,736 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:08:04,934 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:08:13,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:08:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:08:34,935 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:08:39,702 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 25859 2024-12-03T05:08:48,736 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:08:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:09:04,935 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:09:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:09:24,702 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 25859 2024-12-03T05:09:33,737 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:09:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:09:34,935 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:09:37,222 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=693.26 KB, freeSize=879.32 MB, max=880 MB, blockCount=6, accesses=531667, hits=524899, hitRatio=98.73%, , cachingAccesses=525891, cachingHits=524438, cachingHitsRatio=99.72%, evictions=59, evicted=0, evictedPerRun=0.0 2024-12-03T05:09:37,332 DEBUG [master/32beccd76b35:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=7, created chunk count=13, reused chunk count=372, reuseRatio=96.62% 2024-12-03T05:09:37,332 DEBUG [master/32beccd76b35:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-12-03T05:09:39,020 WARN [regionserver/32beccd76b35:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-12-03T05:09:39,047 INFO [regionserver/32beccd76b35:0.Chore.1 {}] regionserver.Replication$ReplicationStatisticsChore(208): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-12-03T05:09:39,937 DEBUG [master/32beccd76b35:0.Chore.1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:09:39,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42173, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:09:39,951 DEBUG [master/32beccd76b35:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T05:09:39,952 DEBUG [master/32beccd76b35:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-03T05:09:39,952 INFO [master/32beccd76b35:0.Chore.1 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-03T05:09:39,952 DEBUG [master/32beccd76b35:0.Chore.1 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-03T05:09:39,953 DEBUG [master/32beccd76b35:0.Chore.1 {}] balancer.BalancerClusterState(204): Hosts are {32beccd76b35=0} racks are {/default-rack=0} 2024-12-03T05:09:39,953 DEBUG [master/32beccd76b35:0.Chore.1 {}] balancer.BalancerClusterState(303): server 0 has 2 regions 2024-12-03T05:09:39,953 DEBUG [master/32beccd76b35:0.Chore.1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T05:09:39,953 INFO [master/32beccd76b35:0.Chore.1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T05:09:39,953 DEBUG [master/32beccd76b35:0.Chore.1 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-03T05:09:39,953 INFO [master/32beccd76b35:0.Chore.1 {}] balancer.StochasticLoadBalancer(357): Not running balancer because only 1 active regionserver(s) 2024-12-03T05:09:39,953 DEBUG [master/32beccd76b35:0.Chore.1 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-03T05:09:39,957 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(219): Skipping table TestAcidGuarantees because normalization is disabled in its table properties and normalization is also disabled at table level by default 2024-12-03T05:09:45,236 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:09:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:10:04,935 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:10:09,703 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 25859 2024-12-03T05:10:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:10:18,737 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:10:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:10:34,936 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:10:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:10:54,703 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 25859 2024-12-03T05:11:03,737 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:11:04,936 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:11:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:11:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:11:34,936 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:11:39,703 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 25859 2024-12-03T05:11:48,738 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 29e602dd3df47e1bb4bce15079b1bb57, had cached 0 bytes from a total of 113862 2024-12-03T05:11:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:12:04,936 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:12:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T05:12:24,703 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 25859 2024-12-03T05:12:31,750 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:12:31,750 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:129) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:12:31,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:12:31,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:12:31,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:12:31,751 ERROR [Time-limited test {}] util.AbstractHBaseTool(152): Error running command-line tool java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Thread.join(Thread.java:1313) ~[?:?] at java.lang.Thread.join(Thread.java:1381) ~[?:?] at org.apache.hadoop.hbase.MultithreadedTestUtil$TestContext.stop(MultithreadedTestUtil.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.AcidGuaranteesTestTool.runTestAtomicity(AcidGuaranteesTestTool.java:390) ~[test-classes/:?] at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:130) ~[test-classes/:?] at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) ~[test-classes/:?] at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:12:31,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:12:31,753 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:12:31,754 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:12:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:12:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T05:12:31,760 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202751760"}]},"ts":"1733202751760"} 2024-12-03T05:12:31,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35291, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:12:31,764 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:12:31,764 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:12:31,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] ====> TEST TIMED OUT. PRINTING THREAD DUMP. <==== Timestamp: 2024-12-03 05:12:31,755 "RPCClient-NioEventLoopGroup-4-6" daemon prio=5 tid=325 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "pool-11-thread-1" prio=5 tid=69 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "AsyncFSWAL-0-hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/MasterData-prefix:32beccd76b35,43423,1733201976038" daemon prio=5 tid=238 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-12" daemon prio=5 tid=335 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "SessionTracker" daemon prio=5 tid=141 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) "RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43423" daemon prio=5 tid=179 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2" daemon prio=5 tid=363 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34719" daemon prio=5 tid=197 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedS2024-12-03T05:12:31,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, UNASSIGN}] ynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "PacketResponder: BP-1710912754-172.17.0.3-1733201972772:blk_1073741834_1010, type=LAST_IN_PIPELINE" daemon prio=5 tid=297 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e/data/data2)" daemon prio=5 tid=117 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656) "pool-1-thread-2" daemon prio=5 tid=15 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:275) at java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test-EventThread" daemon prio=5 tid=190 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) "Registry-endpoints-refresh-end-points" daemon prio=5 tid=302 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.client.RegistryEndpointsRefresher.mainLoop(RegistryEndpointsRefresher.java:92) at app//org.apache.hadoop.hbase.client.RegistryEndpointsRefresher$$Lambda$804/0x00007fa0a4b87228.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "HMaster-EventLoopGroup-1-3" daemon prio=10 tid=304 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "HMaster-EventLoopGroup-1-1" daemon prio=10 tid=154 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-4" daemon prio=5 tid=307 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp700900234-37" daemon prio=5 tid=37 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fa0a4443428.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp700900234-38" daemon prio=5 tid=38 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fa0a4443428.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ProcessThread(sid:0 cport:52673):" daemon prio=5 tid=143 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142) "RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=43423" daemon prio=5 tid=181 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "RPCClient-NioEventLoopGroup-4-8" daemon prio=5 tid=327 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=34719" daemon prio=5 tid=205 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "Reference Handler" daemon prio=10 tid=2 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/java.lang.ref.Reference.waitForReferencePendingList(Native Method) at java.base@17.0.11/java.lang.ref.Reference.processPendingReferences(Reference.java:253) at java.base@17.0.11/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:215) "RPCClient-NioEventLoopGroup-4-11" daemon prio=5 tid=334 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.2024-12-03T05:12:31,767 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, UNASSIGN 0.11/java.lang.Thread.run(Thread.java:840) "SyncThread:0" daemon prio=5 tid=142 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170) "refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e/data/data2/current/BP-1710912754-172.17.0.3-1733201972772" daemon prio=5 tid=123 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "MiniHBaseClusterRegionServer-EventLoopGroup-3-1" daemon prio=10 tid=187 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-7" daemon prio=5 tid=163 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "java.util.concurrent.ThreadPoolExecutor$Worker@27278669[State = -1, empty queue]" daemon prio=5 tid=135 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Notification Thread" daemon prio=9 tid=13 runnable java.lang.Thread.State: RUNNABLE "NIOWorkerThread-16" daemon prio=5 tid=172 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "snapshot-hfile-cleaner-cache-refresher" daemon prio=5 tid=267 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "Socket Reader #1 for port 0" daemon prio=5 tid=95 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476) "MobFileCache #0" daemon prio=5 tid=196 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "master:store-Flusher" daemon prio=5 tid=241 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759) at app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor.flushLoop(MasterRegionFlusherAndCompactor.java:200) at app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor$$Lambda$479/0x00007fa0a49f7118.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e/data/data1/current/BP-1710912754-172.17.0.3-1733201972772" daemon prio=5 tid=124 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "master/32beccd76b35:0:becomeActiveMaster-MemStoreChunkPool Statistics" daemon prio=5 tid=214 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@5114e428" daemon prio=5 tid=72 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:4550) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PEWorker-5" daemon prio=5 tid=251 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) "BootstrapNodeManager" daemon prio=5 tid=258 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "AsyncFSWAL-0-hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d-prefix:32beccd76b35,34719,1733201977102" daemon prio=5 tid=292 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "BP-1710912754-172.17.0.3-1733201972772 heartbeating to localhost/127.0.0.1:37421" daemon prio=5 tid=102 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:771) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:914) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "LruBlockCacheStatsExecutor" daemon prio=5 tid=194 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "M:0;32beccd76b35:43423" daemon prio=5 tid=184 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.util.Sleeper.sle2024-12-03T05:12:31,768 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=152 updating hbase:meta row=29e602dd3df47e1bb4bce15079b1bb57, regionState=CLOSING, regionLocation=32beccd76b35,34719,1733201977102 ep(Sleeper.java:81) at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:64) at app//org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:625) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server listener on 0" daemon prio=5 tid=94 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559) "Hadoop-Metrics-Updater-0" daemon prio=5 tid=58 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1077258357-90" daemon prio=5 tid=90 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@529b70ad" daemon prio=5 tid=71 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:537) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=5 tid=23 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Timer for 'HBase' metrics system" daemon prio=5 tid=3081 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0" daemon prio=5 tid=2335 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "pool-1-thread-1" daemon prio=5 tid=14 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.poll(EPollPort.java:200) at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:281) at java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Parameter Sending Thread for localhost/127.0.0.1:37421" daemon prio=5 tid=110 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) at java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) at app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-5" daemon prio=5 tid=324 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=43423" daemon prio=5 tid=183 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "master/32beccd76b35:0.Chore.1" daemon prio=5 tid=263 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-15" daemon prio=5 tid=342 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Monitor thread for TaskMonitor" daemon prio=5 tid=210 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS:0;32beccd76b35:34719-shortCompactions-0" daemon prio=5 tid=408 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "master:store-WAL-Roller" daemon prio=5 tid=226 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) "RPCClient-NioEventLoopGroup-4-2" daemon prio=5 tid=305 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34719" daemon prio=5 tid=199 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "regionserver/32beccd76b35:0.Chore.1" daemon prio=5 tid=282 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0" daemon prio=5 tid=356 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-9" daemon prio=5 tid=165 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ProcedureDispatcherTimeoutThread" daemon prio=5 tid=252 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) at app//org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread.run(RemoteProcedureDispatcher.java:328) "RegionServerTracker-0" daemon prio=5 tid=275 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Socket Reader #1 for port 0" daemon prio=5 tid=55 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476) "JvmPauseMonitor" daemon prio=5 tid=280 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:148) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Common-Cleaner" daemon prio=8 tid=12 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@17.0.11/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:140) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) at java.base@17.0.11/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:162) "org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor@7cf68930" daemon prio=5 tid=61 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor.run(PendingReconstructionBlocks.java:267) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server idle connection scanner for port 37699" daemon prio=5 tid=96 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "NIOWorkerThread-15" daemon prio=5 tid=171 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=34719" daemon prio=5 tid=206 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.loc2024-12-03T05:12:31,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29e602dd3df47e1bb4bce15079b1bb57, UNASSIGN because future has completed ks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "Time-limited test-SendThread(127.0.0.1:52673)" daemon prio=5 tid=155 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) "NIOWorkerThread-8" daemon prio=5 tid=164 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Client (1102635514) connection to localhost/127.0.0.1:37421 from jenkins" daemon prio=5 tid=109 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) at app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) "LeaseRenewer:jenkins@localhost:37421" daemon prio=5 tid=147 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "DataXceiver for client DFSClient_NONMAPREDUCE_-1947188745_22 at /127.0.0.1:38040 [Receiving block BP-1710912754-172.17.0.3-1733201972772:blk_1073741830_1006]" daemon prio=5 tid=236 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "DatanodeAdminMonitor-0" daemon prio=5 tid=62 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test-EventThread" daemon prio=5 tid=156 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) "OldWALsCleaner-0" daemon prio=5 tid=261 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSync2024-12-03T05:12:31,770 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:12:31,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; CloseRegionProcedure 29e602dd3df47e1bb4bce15079b1bb57, server=32beccd76b35,34719,1733201977102}] hronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner.deleteFile(LogCleaner.java:172) at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner.lambda$createOldWalsCleaner$1(LogCleaner.java:152) at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner$$Lambda$598/0x00007fa0a4ac7d50.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "main" prio=5 tid=1 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/java.lang.Thread.dumpThreads(Native Method) at java.base@17.0.11/java.lang.Thread.getAllStackTraces(Thread.java:1671) at app//org.apache.hadoop.hbase.TimedOutTestsListener.buildThreadDump(TimedOutTestsListener.java:92) at app//org.apache.hadoop.hbase.TimedOutTestsListener.buildThreadDiagnosticString(TimedOutTestsListener.java:78) at app//org.apache.hadoop.hbase.TimedOutTestsListener.testFailure(TimedOutTestsListener.java:65) at app//org.junit.runner.notification.SynchronizedRunListener.testFailure(SynchronizedRunListener.java:94) at app//org.junit.runner.notification.RunNotifier$6.notifyListener(RunNotifier.java:177) at app//org.junit.runner.notification.RunNotifier$SafeNotifier.run(RunNotifier.java:72) at app//org.junit.runner.notification.RunNotifier.fireTestFailures(RunNotifier.java:173) at app//org.junit.runner.notification.RunNotifier.fireTestFailure(RunNotifier.java:167) at app//org.apache.maven.surefire.common.junit4.Notifier.fireTestFailure(Notifier.java:100) at app//org.junit.internal.runners.model.EachTestNotifier.addFailure(EachTestNotifier.java:23) at app//org.junit.internal.runners.model.EachTestNotifier.addMultipleFailureException(EachTestNotifier.java:29) at app//org.junit.internal.runners.model.EachTestNotifier.addFailure(EachTestNotifier.java:21) at app//org.junit.runners.ParentRunner.run(ParentRunner.java:419) at app//org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:316) at app//org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:240) at app//org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:214) at app//org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:155) at app//org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:385) at app//org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162) at app//org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:507) at app//org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:495) "IPC Server handler 0 on default port 37421" daemon prio=5 tid=64 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34719" daemon prio=5 tid=198 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "LeaseRenewer:jenkins.hfs.0@localhost:37421" daemon prio=5 tid=291 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) at app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=43423" daemon prio=5 tid=180 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "PEWorker-3" daemon prio=5 tid=249 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) "Signal Dispatcher" daemon prio=9 tid=4 runnable java.lang.Thread.State: RUNNABLE "PEWorker-1" daemon prio=5 tid=247 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) "NIOWorkerThread-2" daemon prio=5 tid=157 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS:0;32beccd76b35:34719-longCompactions-0" daemon prio=5 tid=281 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:101) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423" daemon prio=5 tid=174 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230) at app//org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch$CompatibilityLatch.await(ProcedurePrepareLatch.java:114) at app//org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2833) at app//org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at app//org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) at app//org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) at app//org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at app//org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at app//org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "RPCClient-NioEventLoopGroup-4-16" daemon prio=5 tid=343 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-4" daemon prio=5 tid=159 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test.LruBlockCache.EvictionThread" daemon prio=5 tid=192 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:957) "Idle-Rpc-Conn-Sweeper-pool-0" daemon prio=5 tid=244 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "FSEditLogAsync" daemon prio=5 tid=53 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:241) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:250) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.util.JvmPauseMonitor$Monitor@77d52ff6" daemon prio=5 tid=93 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server listener on 0" daemon prio=5 tid=54 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559) "RPCClient-NioEventLoopGroup-4-7" daemon prio=5 tid=326 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "FsDatasetAsyncDiskServiceFixer" daemon prio=5 tid=136 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer.run(HBaseTestingUtil.java:576) "AsyncFSWAL-0-hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d-prefix:32beccd76b35,34719,1733201977102.meta" daemon prio=5 tid=298 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Command processor" daemon prio=5 tid=101 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) "Thread-2364" daemon prio=5 tid=2610 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1864) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1898) at java.base@17.0.11/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2072) at app//org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) at app//org.apache.hadoop.hbase.client.AdminOverAsyncAdmin.flush(AdminOverAsyncAdmin.java:259) at app//org.apache.hadoop.hbase.AcidGuaranteesTestTool$1.doAnAction(AcidGuaranteesTestTool.java:355) at app//org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at app//org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) "RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=43423" daemon prio=5 tid=182 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "pool-19-thread-1" prio=5 tid=103 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PEWorker-4" daemon prio=5 tid=250 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.String.valueOf(String.java:4220) at java.base@17.0.11/java.lang.StringBuilder.append(StringBuilder.java:173) at app//io.opentelemetry.context.ArrayBasedContext.toString(ArrayBasedContext.java:89) at java.base@17.0.11/java.lang.String.valueOf(String.java:4220) at java.base@17.0.11/java.lang.StringBuilder.append(StringBuilder.java:173) at app//io.opentelemetry.context.StrictContextStorage$CallerStackTrace.(StrictContextStorage.java:230) at app//io.opentelemetry.context.StrictContextStorage.attach(StrictContextStorage.java:67) at app//io.opentelemetry.context.Context.makeCurrent(Context.java:202) at app//io.opentelemetry.context.ImplicitContextKeyed.makeCurrent(ImplicitContextKeyed.java:33) at app//org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:215) at app//org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at app//org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.lambda$update$6(RegionProcedureStore.java:367) at app//org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore$$Lambda$544/0x00007fa0a4a7bce0.update(Unknown Source) at app//org.apache.hadoop.hbase.master.region.MasterRegion.update(MasterRegion.java:148) at app//org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.lambda$update$7(RegionProcedureStore.java:367) at app//org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore$$Lambda$542/0x00007fa0a4a71e78.run(Unknown Source) at app//org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.runWithoutRpcCall(RegionProcedureStore.java:315) at app//org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.update(RegionProcedureStore.java:364) at app//org.apache.hadoop.hbase.procedure2.Procedure.doAcquireLock(Procedure.java:1014) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.acquireLock(ProcedureExecutor.java:1540) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1500) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread$$Lambda$593/0x00007fa0a4ab4730.call(Unknown Source) at app//org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) "NIOWorkerThread-14" daemon prio=5 tid=170 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-10" daemon prio=5 tid=166 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=34719" daemon prio=5 tid=204 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "Session-HouseKeeper-5d1e5c6b-1" prio=5 tid=91 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-14" daemon prio=5 tid=339 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RedundancyMonitor" daemon prio=5 tid=47 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at java.base@17.0.11/java.lang.Thread.sleep(Thread.java:344) at java.base@17.0.11/java.util.concurrent.TimeUnit.sleep(TimeUnit.java:446) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$RedundancyMonitor.run(BlockManager.java:5352) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ConnnectionExpirer" daemon prio=5 tid=137 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:554) "PacketResponder: BP-1710912754-172.17.0.3-1733201972772:blk_1073741830_1006, type=LAST_IN_PIPELINE" daemon prio=5 tid=237 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "SSL Certificates Store Monitor" daemon prio=5 tid=25 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "NIOWorkerThread-13" daemon prio=5 tid=169 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-6" daemon prio=5 tid=162 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-11" daemon prio=5 tid=167 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34719" daemon prio=5 tid=202 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "32beccd76b35:43423" daemon prio=5 tid=253 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.waitOnAssignQueue(AssignmentManager.java:2398) at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.processAssignQueue(AssignmentManager.java:2420) at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager$1.run(AssignmentManager.java:2360) "org.apache.hadoop.util.JvmPauseMonitor$Monitor@ba47c44" daemon prio=5 tid=34 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34719" daemon prio=5 tid=200 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "Time-limited test.named-queue-events-pool-0" daemon prio=5 tid=186 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) at app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) at app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) at app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-10" daemon prio=5 tid=331 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp700900234-43" daemon prio=5 tid=43 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 1 on default port 37421" daemon prio=5 tid=65 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "master/32beccd76b35:0:becomeActiveMaster-HFileCleaner.small.0-1733201978891" daemon prio=5 tid=269 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:285) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner$2.run(HFileCleaner.java:269) "RpcClient-timer-pool-0" daemon prio=5 tid=243 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-12" daemon prio=5 tid=168 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test" daemon prio=5 tid=22 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1866) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1939) at java.base@17.0.11/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2095) at app//org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:196) at app//org.apache.hadoop.hbase.client.Admin.disableTable(Admin.java:397) at app//org.apache.hadoop.hbase.HBaseTestingUtil.deleteTable(HBaseTestingUtil.java:1551) at app//org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDown(AcidGuaranteesTestBase.java:86) at java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) at app//org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at app//org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at app//org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at app//org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at app//org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at app//org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at app//org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at app//org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at app//org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at app//org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at app//org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at app//org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at app//org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at app//org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at app//org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at app//org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at app//org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at app//org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at app//org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 3 on default port 37699" daemon prio=5 tid=107 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "NIOServerCxnFactory.AcceptThread:localhost/127.0.0.1:52673" daemon prio=5 tid=140 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181) "regionserver/32beccd76b35:0.logRoller" daemon prio=5 tid=284 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) "master/32beccd76b35:0:becomeActiveMaster-HFileCleaner.large.0-1733201978891" daemon prio=5 tid=268 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:101) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:285) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner$1.run(HFileCleaner.java:254) "IPC Server handler 0 on default port 37699" daemon prio=5 tid=104 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "qtp700900234-44" daemon prio=5 tid=44 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "zk-event-processor-pool-0" daemon prio=5 tid=191 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "SnapshotHandlerChoreCleaner" daemon prio=5 tid=255 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1077258357-88-acceptor-0@4ea3f0b3-ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39265}" daemon prio=3 tid=88 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "surefire-forkedjvm-stream-flusher" daemon prio=5 tid=16 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "normalizer-worker-0" daemon prio=5 tid=254 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorkQueue.take(RegionNormalizerWorkQueue.java:146) at app//org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorker.run(RegionNormalizerWorker.java:191) at java.base@17.0.11/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS_CLOSE_REGION-regionserver/32beccd76b35:0-0" daemon prio=5 tid=783 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "surefire-forkedjvm-command-thread" daemon prio=5 tid=18 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230) at java.base@17.0.11/sun.nio.ch.PendingFuture.get(PendingFuture.java:178) at app//org.apache.maven.surefire.api.util.internal.Channels$2.read(Channels.java:127) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at app//org.apache.maven.surefire.api.util.internal.Channels$3.readImpl(Channels.java:169) at app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleReadableChannel.read(AbstractNoninterruptibleReadableChannel.java:50) at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:430) at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:419) at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.readMessageType(AbstractStreamDecoder.java:116) at app//org.apache.maven.surefire.booter.stream.CommandDecoder.decode(CommandDecoder.java:77) at app//org.apache.maven.surefire.booter.spi.CommandChannelDecoder.decode(CommandChannelDecoder.java:60) at app//org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:290) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-1" daemon prio=5 tid=145 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "DataXceiver for client DFSClient_NONMAPREDUCE_-775871842_22 at /127.0.0.1:38060 [Receiving block BP-1710912754-172.17.0.3-1733201972772:blk_1073741833_1009]" daemon prio=5 tid=289 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 4 on default port 37699" daemon prio=5 tid=108 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "qtp700900234-39" daemon prio=5 tid=39 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fa0a4443428.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 2 on default port 37421" daemon prio=5 tid=66 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "IPC Server Responder" daemon prio=5 tid=57 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733) at app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716) "qtp1077258357-87" daemon prio=5 tid=87 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fa0a4443428.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1" daemon prio=5 tid=361 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test-SendThread(127.0.0.1:52673)" daemon prio=5 tid=189 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) "region-location-0" daemon prio=5 tid=301 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-9" daemon prio=5 tid=330 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "HBase-Metrics2-1" daemon prio=5 tid=153 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS_OPEN_META-regionserver/32beccd76b35:0-0" daemon prio=5 tid=295 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "pool-6-thread-1" prio=5 tid=46 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-13" daemon prio=5 tid=338 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "DataXceiver for client DFSClient_NONMAPREDUCE_-775871842_22 at /127.0.0.1:38076 [Receiving block BP-1710912754-172.17.0.3-1733201972772:blk_1073741834_1010]" daemon prio=5 tid=296 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp700900234-41-acceptor-0@7f161b54-ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:42397}" daemon prio=3 tid=41 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43423" daemon prio=5 tid=177 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9710aa0b-628a-b683-c011-d2851e76e40b/cluster_c2820f1d-06ea-6af3-ef0f-23177978988e/data/data1)" daemon prio=5 tid=116 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@dc6fb" daemon prio=5 tid=73 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:4592) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "regionserver/32beccd76b35:0.leaseChecker" daemon prio=5 tid=283 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) "qtp700900234-40" daemon prio=5 tid=40 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fa0a4443428.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 3 on default port 37421" daemon prio=5 tid=67 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "Hadoop-Metrics-Updater-0" daemon prio=5 tid=98 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1077258357-89" daemon prio=5 tid=89 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-1" daemon prio=5 tid=303 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOServerCxnFactory.SelectorThread-1" daemon prio=5 tid=139 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368) "MemStoreFlusher.0" daemon prio=5 tid=286 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:77) at app//org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:323) "Finalizer" daemon prio=8 tid=3 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at java.base@17.0.11/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:172) "CacheReplicationMonitor(992399539)" daemon prio=5 tid=75 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759) at app//org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:186) "Session-HouseKeeper-4385d0c6-1" prio=5 tid=45 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43423" daemon prio=5 tid=178 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "ProcExecTimeout" daemon prio=5 tid=245 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:279) at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) at app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) "weak-ref-cleaner-strictcontextstorage" daemon prio=1 tid=152 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-5" daemon prio=5 tid=161 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@5fb37b07" daemon prio=5 tid=85 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) at java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126) at app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PacketResponder: BP-1710912754-172.17.0.3-1733201972772:blk_1073741833_1009, type=LAST_IN_PIPELINE" daemon prio=5 tid=290 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS_OPEN_REGION-regionserver/32beccd76b35:0-0" daemon prio=5 tid=319 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server Responder" daemon prio=5 tid=97 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733) at app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716) "HMaster-EventLoopGroup-1-2" daemon prio=10 tid=260 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-3" daemon prio=5 tid=158 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34719" daemon prio=5 tid=201 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "MiniHBaseClusterRegionServer-EventLoopGroup-3-2" daemon prio=10 tid=235 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100" daemon prio=5 tid=35 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) "org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@6578e0d7" daemon prio=5 tid=49 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:563) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 1 on default port 37699" daemon prio=5 tid=105 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "pool-17-thread-1" prio=5 tid=86 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-3" daemon prio=5 tid=306 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 2 on default port 37699" daemon prio=5 tid=106 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "org.apache.hadoop.hdfs.PeerCache@97f8edd" daemon prio=5 tid=216 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) at app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Block report processor" daemon prio=5 tid=51 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.processQueue(BlockManager.java:5627) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.run(BlockManager.java:5614) "zk-event-processor-pool-0" daemon prio=5 tid=160 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PEWorker-2" daemon prio=5 tid=248 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) "RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43423" daemon prio=5 tid=175 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "MarkedDeleteBlockScrubberThread" daemon prio=5 tid=48 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$MarkedDeleteBlockScrubber.run(BlockManager.java:5326) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@1b92560e" daemon prio=5 tid=74 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:4689) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp700900234-42" daemon prio=5 tid=42 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server idle connection scanner for port 37421" daemon prio=5 tid=56 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "RequestThrottler" daemon prio=5 tid=144 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147) "IPC Server handler 4 on default port 37421" daemon prio=5 tid=68 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "pool-5-thread-1" prio=5 tid=36 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=34719" daemon prio=5 tid=203 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43423" daemon prio=5 tid=176 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) "nioEventLoopGroup-2-1" prio=10 tid=92 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) at app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOServerCxnFactory.SelectorThread-0" daemon prio=5 tid=138 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368) "Async-Client-Retry-Timer-pool-0" daemon prio=5 tid=242 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "pool-14-thread-1" daemon prio=5 tid=132 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "regionserver/32beccd76b35:0.procedureResultReporter" daemon prio=5 tid=285 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) "MiniHBaseClusterRegionServer-EventLoopGroup-3-3" daemon prio=10 tid=294 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "WorkerMonitor" daemon prio=5 tid=246 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:279) at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) at app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) "master/32beccd76b35:0:becomeActiveMaster-MemStoreChunkPool Statistics" daemon prio=5 tid=212 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS:0;32beccd76b35:34719" daemon prio=5 tid=207 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:81) at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:64) at app//org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:906) at app//org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at app//org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) at java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base@17.0.11/javax.security.auth.Subject.doAs(Subject.java:376) at app//org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at app//org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at app//org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:12:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T05:12:31,923 DEBUG [RSProcedureDispatcher-pool-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T05:12:31,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53073, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T05:12:31,924 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(122): Close 29e602dd3df47e1bb4bce15079b1bb57 2024-12-03T05:12:31,924 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:12:31,924 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1722): Closing 29e602dd3df47e1bb4bce15079b1bb57, disabling compactions & flushes 2024-12-03T05:12:31,924 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:12:31,924 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:12:31,924 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. after waiting 0 ms 2024-12-03T05:12:31,925 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202121679.29e602dd3df47e1bb4bce15079b1bb57. 2024-12-03T05:12:31,925 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(2902): Flushing 29e602dd3df47e1bb4bce15079b1bb57 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-12-03T05:12:31,935 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37421/user/jenkins/test-data/614b8601-affb-a1a0-f9bd-79049476d81d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c4017d70de884cc1abfea3dacdc64ebf_29e602dd3df47e1bb4bce15079b1bb57 is 50, key is test_row_1/A:col10/1733202143940/Put/seqid=0 2024-12-03T05:12:31,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41841 is added to blk_1073742449_1625 (size=9914) 2024-12-03T05:12:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150